[Oxide] Automatic content detection (#11173)

* resolve all _existing_ content paths

* pin `@napi-rs/cli`

* WIP: Log all resolved content files/globs

* only filter out raw changed content in non-auto mode

* skip parseCandidateFiles cache in `auto` mode

* improve algorithm of detecting content paths

1. Files in the root should be listed statically instead of using globs.
2. Files and folders in special known direct child folders should be
   listed statically instead of using globs (e.g.: `public`). This is
   because these special folders are often used to store generated AND
   source files at the same time. Using globs could trigger infinite
   loops because we are watching and acting upon dist files.
3. All file extensions found in the project, should be used in the globs
   in addition to a known set of extensions.
4. Direct folders seen from the root, can use the glob syntax
   `<root>/src/**/*.{...known-extensions}`

* inline wanted-extensions

Not 100% convinced yet, but seems cleaner so far.

* ensure writing an file also makes the parent folder(s)

* add integration tests for the auto content feature

* add pnpm and bun lock files

* Revert "inline wanted-extensions"

This reverts commit 879c1248524e84216125f4a24e0160b40736333a.

* sort binary-extensions and add lockb

* sort + add `lock` to ignored extensions

* drop `yarn.lock`, because lock extensions are already covered

* group template extensions

This will make it a bit easier to organize in the future.

* drop empty lines and commented lines from template-extensions

* skip the config path when resolving template files

The config file will automatically trigger a rebuild when this file is
changed. However, this should not be part of the template files because
that could cause additional css that's not being used.

* make `auto content` the default in the oxide engine

- In the oxide engine, the default `content: []` will be dropped from
  the default configuration (config.simple.js, config.full.js).
- If you have `content: []` or `content: { files: [] }` then the auto
  content feature won't be active. However if those arrays are empty a
  warning will still be shown. Adding files/globs or dropping the
  `content` section completely will enable auto content.

* only test the auto content integration test in the oxide engine

* set `content.files` to `auto` instead of using `auto: boolean`

This way we don't run into the issue where the `config.content.files` is
set and the `config.content.auto` is set to true.

* drop log

* ensure we validate the config in the CLI

* show experimental warning for automatic content detection

* use cached version of the getCandidateFiles instead of bypassing it

* use `is_empty()` shorthand

Thanks, Clippy!

* add test to ensure nested ignored folders are not scanned

* add `tempfile` for tests

* add auto content tests in Rust

* refactor auto content detection

This will also make sure that if we have (deeply) nested ignored
folders, then we won't use deeply nested globs (**/*.{js,html}) for the
parent(s) of the nested ignored folders but instead use a shallow glob
for each directory (*/*.{js,html}).

Then each sibling directory of the parent can use deeply nested globs
again except for the direct parent.

* use consistent comments

* ensure ignored static listed files are not present

* improve performance by ~30x

On a big test project this goes from ~6s to ~200ms

* improve performance by ~5x

We started with a ~6s duration
Then in the previous commit, we improved it by ~30x and it went down to
~200ms
Now with this change, it takes about ~40ms. That's another ~5x
improvement.

Or in total a ~150x improvement.

* ensure nested folders in `public/` are also explicitly listed

* add shortcut for normalizing files

This is only called once so won't do anything to the main performance of
Tailwind CSS. But always nice to make small performance improvements!

* run Rust tests in CI

* fix lint warnings

* update changelog

* Update CHANGELOG.md

---------

Co-authored-by: Robin Malfait <malfait.robin@gmail.com>

src/cli/build/plugin.js

@@ -19,6 +19,7 @@ import { findAtConfigPath } from '../../lib/findAtConfigPath.js'
 import log from '../../util/log'
 import { loadConfig } from '../../lib/load-config'
 import getModuleDependencies from '../../lib/getModuleDependencies'
+import { validateConfig } from '../../util/validateConfig'
 
 /**
  *
@@ -161,7 +162,13 @@ let state = {
     }
 
     // @ts-ignore
-    this.configBag.config = resolveConfig(this.configBag.config, { content: { files: [] } })
+    if (__OXIDE__) {
+      this.configBag.config = validateConfig(resolveConfig(this.configBag.config))
+    } else {
+      this.configBag.config = validateConfig(
+        resolveConfig(this.configBag.config, { content: { files: [] } })
+      )
+    }
 
     // Override content files if `--content` has been passed explicitly
     if (content?.length > 0) {

src/cli/init/index.js

@@ -38,6 +38,11 @@ export function init(args) {
       'utf8'
     )
 
+    // Drop `content` in the oxide engine to promote auto content
+    if (__OXIDE__) {
+      stubContentsFile = stubContentsFile.replace(/\s*content: \[\],\n/, '')
+    }
+
     // Change colors import
     stubContentsFile = stubContentsFile.replace('../colors', 'tailwindcss/colors')
 

src/lib/content.js

@@ -11,6 +11,28 @@ import { env } from './sharedState'
 /** @typedef {import('../../types/config.js').RawFile} RawFile */
 /** @typedef {import('../../types/config.js').FilePath} FilePath */
 
+/*
+ * @param {import('tailwindcss').Config} tailwindConfig
+ * @param {{skip:string[]}} options
+ * @returns {ContentPath[]}
+ */
+function resolveContentFiles(tailwindConfig, { skip = [] } = {}) {
+  if (tailwindConfig.content.files === 'auto' && __OXIDE__) {
+    env.DEBUG && console.time('Calculating resolve content paths')
+    tailwindConfig.content.files = require('@tailwindcss/oxide').resolveContentPaths({
+      base: process.cwd(),
+    })
+    if (skip.length > 0) {
+      tailwindConfig.content.files = tailwindConfig.content.files.filter(
+        (filePath) => !skip.includes(filePath)
+      )
+    }
+    env.DEBUG && console.timeEnd('Calculating resolve content paths')
+  }
+
+  return tailwindConfig.content.files
+}
+
 /**
  * @typedef {object} ContentPath
  * @property {string} original
@@ -32,7 +54,9 @@ import { env } from './sharedState'
  * @returns {ContentPath[]}
  */
 export function parseCandidateFiles(context, tailwindConfig) {
-  let files = tailwindConfig.content.files
+  let files = resolveContentFiles(tailwindConfig, {
+    skip: [context.userConfigPath],
+  })
 
   // Normalize the file globs
   files = files.filter((filePath) => typeof filePath === 'string')
@@ -167,9 +191,12 @@ function resolvePathSymlinks(contentPath) {
  * @returns {[{ content: string, extension: string }[], Map<string, number>]}
  */
 export function resolvedChangedContent(context, candidateFiles, fileModifiedMap) {
-  let changedContent = context.tailwindConfig.content.files
-    .filter((item) => typeof item.raw === 'string')
-    .map(({ raw, extension = 'html' }) => ({ content: raw, extension }))
+  let changedContent =
+    context.tailwindConfig.content.files === 'auto' && __OXIDE__
+      ? []
+      : context.tailwindConfig.content.files
+          .filter((item) => typeof item.raw === 'string')
+          .map(({ raw, extension = 'html' }) => ({ content: raw, extension }))
 
   let [changedFiles, mTimesToCommit] = resolveChangedFiles(candidateFiles, fileModifiedMap)
 

src/oxide/cli/build/plugin.ts

@@ -18,6 +18,7 @@ import log from '../../../util/log'
 import { loadConfig } from '../../../lib/load-config'
 import getModuleDependencies from '../../../lib/getModuleDependencies'
 import type { Config } from '../../../../types'
+import { validateConfig } from '../../../util/validateConfig'
 
 /**
  *
@@ -160,7 +161,13 @@ let state = {
     }
 
     // @ts-ignore
-    this.configBag.config = resolveConfig(this.configBag.config, { content: { files: [] } })
+    if (__OXIDE__) {
+      this.configBag.config = validateConfig(resolveConfig(this.configBag.config))
+    } else {
+      this.configBag.config = validateConfig(
+        resolveConfig(this.configBag.config, { content: { files: [] } })
+      )
+    }
 
     // Override content files if `--content` has been passed explicitly
     if (content?.length > 0) {

src/oxide/cli/init/index.ts

@@ -36,6 +36,11 @@ export function init(args) {
       'utf8'
     )
 
+    // Drop `content` in the oxide engine to promote auto content
+    if (__OXIDE__) {
+      stubContentsFile = stubContentsFile.replace(/\s*content: \[\],\n/, '')
+    }
+
     // Change colors import
     stubContentsFile = stubContentsFile.replace('../colors', 'tailwindcss/colors')
 

src/util/getAllConfigs.js

@@ -2,7 +2,14 @@ import defaultFullConfig from '../../stubs/config.full.js'
 import { flagEnabled } from '../featureFlags'
 
 export default function getAllConfigs(config) {
-  const configs = (config?.presets ?? [defaultFullConfig])
+  const configs = (
+    config?.presets ?? [
+      __OXIDE__
+        ? // Drop `content` in the oxide engine to promote auto content
+          Object.assign({}, defaultFullConfig, { content: 'auto' })
+        : defaultFullConfig,
+    ]
+  )
     .slice()
     .reverse()
     .flatMap((preset) => getAllConfigs(preset instanceof Function ? preset() : preset))

src/util/normalizeConfig.js

@@ -18,6 +18,10 @@ export function normalizeConfig(config) {
    *   }
    */
   let valid = (() => {
+    if (config.content === 'auto') {
+      return true
+    }
+
     // `config.purge` should not exist anymore
     if (config.purge) {
       return false
@@ -181,6 +185,31 @@ export function normalizeConfig(config) {
     config.prefix = config.prefix ?? ''
   }
 
+  let auto = (() => {
+    // Config still has a `purge` option (for backwards compatibility), auto content should not be
+    // used
+    if (config.purge) return false
+
+    //
+    if (config.content === 'auto') return true
+
+    // We don't have content at all, auto content should be used
+    if (config.content === undefined) return true
+
+    // We do have content as an object, but we don't have any files defined, auto content should
+    // be used
+    if (
+      typeof config.content === 'object' &&
+      config.content !== null &&
+      !Array.isArray(config.content)
+    ) {
+      return config.content.files === undefined
+    }
+
+    // We do have content defined, auto content should not be used
+    return false
+  })()
+
   // Normalize the `content`
   config.content = {
     relative: (() => {
@@ -193,17 +222,20 @@ export function normalizeConfig(config) {
       return flagEnabled(config, 'relativeContentPathsByDefault')
     })(),
 
-    files: (() => {
-      let { content, purge } = config
+    files: auto
+      ? 'auto'
+      : (() => {
+          let { content, purge } = config
 
-      if (Array.isArray(purge)) return purge
-      if (Array.isArray(purge?.content)) return purge.content
-      if (Array.isArray(content)) return content
-      if (Array.isArray(content?.content)) return content.content
-      if (Array.isArray(content?.files)) return content.files
+          if (content === undefined && purge === undefined) return []
+          if (Array.isArray(purge)) return purge
+          if (Array.isArray(purge?.content)) return purge.content
+          if (Array.isArray(content)) return content
+          if (Array.isArray(content?.content)) return content.content
+          if (Array.isArray(content?.files)) return content.files
 
-      return []
-    })(),
+          return []
+        })(),
 
     extract: (() => {
       let extract = (() => {
@@ -286,14 +318,16 @@ export function normalizeConfig(config) {
 
   // Validate globs to prevent bogus globs.
   // E.g.: `./src/*.{html}` is invalid, the `{html}` should just be `html`
-  for (let file of config.content.files) {
-    if (typeof file === 'string' && /{([^,]*?)}/g.test(file)) {
-      log.warn('invalid-glob-braces', [
-        `The glob pattern ${dim(file)} in your Tailwind CSS configuration is invalid.`,
-        `Update it to ${dim(file.replace(/{([^,]*?)}/g, '$1'))} to silence this warning.`,
-        // TODO: Add https://tw.wtf/invalid-glob-braces
-      ])
-      break
+  if (config.content.files !== 'auto') {
+    for (let file of config.content.files) {
+      if (typeof file === 'string' && /{([^,]*?)}/g.test(file)) {
+        log.warn('invalid-glob-braces', [
+          `The glob pattern ${dim(file)} in your Tailwind CSS configuration is invalid.`,
+          `Update it to ${dim(file.replace(/{([^,]*?)}/g, '$1'))} to silence this warning.`,
+          // TODO: Add https://tw.wtf/invalid-glob-braces
+        ])
+        break
+      }
     }
   }
 

src/util/validateConfig.js

@@ -1,7 +1,7 @@
 import log from './log'
 
 export function validateConfig(config) {
-  if (config.content.files.length === 0) {
+  if (config.content.files !== 'auto' && config.content.files.length === 0) {
     log.warn('content-problems', [
       'The `content` option in your Tailwind CSS configuration is missing or empty.',
       'Configure your content sources or your generated CSS will be missing styles.',
@@ -9,6 +9,13 @@ export function validateConfig(config) {
     ])
   }
 
+  if (config.content.files === 'auto') {
+    log.warn('auto-content-experimental', [
+      'Automatic content detection in Tailwind CSS is currently in experimental preview.',
+      'Preview features are not covered by semver, and may change at any time.',
+    ])
+  }
+
   // Warn if the line-clamp plugin is installed
   try {
     let plugin = require('@tailwindcss/line-clamp')