diff --git a/crates/node/src/lib.rs b/crates/node/src/lib.rs index c494f008a..10d8f99d7 100644 --- a/crates/node/src/lib.rs +++ b/crates/node/src/lib.rs @@ -1,6 +1,10 @@ +use utf16::IndexConverter; + #[macro_use] extern crate napi_derive; +mod utf16; + #[derive(Debug, Clone)] #[napi(object)] pub struct ChangedContent { @@ -123,13 +127,25 @@ impl Scanner { &mut self, input: ChangedContent, ) -> Vec { + let content = input.content.unwrap_or_else(|| { + std::fs::read_to_string(&input.file.unwrap()).expect("Failed to read file") + }); + + let input = ChangedContent { + file: None, + content: Some(content.clone()), + extension: input.extension, + }; + + let mut utf16_idx = IndexConverter::new(&content[..]); + self .scanner .get_candidates_with_positions(input.into()) .into_iter() .map(|(candidate, position)| CandidateWithPosition { candidate, - position: position as i64, + position: utf16_idx.get(position), }) .collect() } diff --git a/crates/node/src/utf16.rs b/crates/node/src/utf16.rs new file mode 100644 index 000000000..7cdebd9fd --- /dev/null +++ b/crates/node/src/utf16.rs @@ -0,0 +1,100 @@ +/// The `IndexConverter` is used to convert UTF-8 *BYTE* indexes to UTF-16 +/// *character* indexes +#[derive(Clone)] +pub struct IndexConverter<'a> { + input: &'a str, + curr_utf8: usize, + curr_utf16: usize, +} + +impl<'a> IndexConverter<'a> { + pub fn new(input: &'a str) -> Self { + Self { + input, + curr_utf8: 0, + curr_utf16: 0, + } + } + + pub fn get(&mut self, pos: usize) -> i64 { + #[cfg(debug_assertions)] + if self.curr_utf8 > self.input.len() { + panic!("curr_utf8 points past the end of the input string"); + } + + if pos < self.curr_utf8 { + self.curr_utf8 = 0; + self.curr_utf16 = 0; + } + + // SAFETY: No matter what `pos` is passed into this function `curr_utf8` + // will only ever be incremented up to the length of the input string. + // + // This eliminates a "potential" panic that cannot actually happen + let slice = unsafe { + self.input.get_unchecked(self.curr_utf8..) + }; + + for c in slice.chars() { + if self.curr_utf8 >= pos { + break + } + + self.curr_utf8 += c.len_utf8(); + self.curr_utf16 += c.len_utf16(); + } + + return self.curr_utf16 as i64; + } +} + +#[cfg(test)] +mod test { + use super::*; + use std::collections::HashMap; + + #[test] + fn test_index_converter() { + let mut converter = IndexConverter::new("Hello 🔥🥳 world!"); + + let map = HashMap::from([ + // hello + (0, 0), + (1, 1), + (2, 2), + (3, 3), + (4, 4), + (5, 5), + (6, 6), + + // inside the 🔥 + (7, 8), + (8, 8), + (9, 8), + (10, 8), + + // inside the 🥳 + (11, 10), + (12, 10), + (13, 10), + (14, 10), + + // world! + (15, 11), + (16, 12), + (17, 13), + (18, 14), + (19, 15), + (20, 16), + (21, 17), + + // Past the end should return the last utf-16 character index + (22, 17), + (100, 17), + ]); + + for (idx_utf8, idx_utf16) in map { + assert_eq!(converter.get(idx_utf8), idx_utf16); + } + } +} diff --git a/packages/@tailwindcss-upgrade/package.json b/packages/@tailwindcss-upgrade/package.json index 195377742..d74a85796 100644 --- a/packages/@tailwindcss-upgrade/package.json +++ b/packages/@tailwindcss-upgrade/package.json @@ -38,7 +38,6 @@ "postcss-import": "^16.1.0", "postcss-selector-parser": "^6.1.2", "prettier": "^3.3.3", - "string-byte-slice": "^3.0.0", "tailwindcss": "workspace:^", "tree-sitter": "^0.21.1", "tree-sitter-typescript": "^0.23.0" diff --git a/packages/@tailwindcss-upgrade/src/template/candidates.test.ts b/packages/@tailwindcss-upgrade/src/template/candidates.test.ts index 4157441c8..dbab76ddd 100644 --- a/packages/@tailwindcss-upgrade/src/template/candidates.test.ts +++ b/packages/@tailwindcss-upgrade/src/template/candidates.test.ts @@ -1,6 +1,7 @@ import { __unstable__loadDesignSystem } from '@tailwindcss/node' import { describe, expect, test } from 'vitest' -import { extractRawCandidates, printCandidate, replaceCandidateInContent } from './candidates' +import { extractRawCandidates, printCandidate } from './candidates' +import { spliceChangesIntoString } from './splice-changes-into-string' let html = String.raw @@ -66,13 +67,20 @@ test('replaces the right positions for a candidate', async () => { ({ rawCandidate }) => designSystem.parseCandidate(rawCandidate).length > 0, )! - expect(replaceCandidateInContent(content, 'flex', candidate.start, candidate.end)) - .toMatchInlineSnapshot(` + let migrated = spliceChangesIntoString(content, [ + { + start: candidate.start, + end: candidate.end, + replacement: 'flex', + }, + ]) + + expect(migrated).toMatchInlineSnapshot(` + " +

🤠👋

+
" -

🤠👋

-
- " - `) + `) }) const candidates = [ diff --git a/packages/@tailwindcss-upgrade/src/template/candidates.ts b/packages/@tailwindcss-upgrade/src/template/candidates.ts index be08477ee..87d4e345e 100644 --- a/packages/@tailwindcss-upgrade/src/template/candidates.ts +++ b/packages/@tailwindcss-upgrade/src/template/candidates.ts @@ -1,5 +1,4 @@ import { Scanner } from '@tailwindcss/oxide' -import stringByteSlice from 'string-byte-slice' import type { Candidate, Variant } from '../../../tailwindcss/src/candidate' import type { DesignSystem } from '../../../tailwindcss/src/design-system' @@ -139,12 +138,3 @@ function escapeArbitrary(input: string) { .replaceAll('_', String.raw`\_`) // Escape underscores to keep them as-is .replaceAll(' ', '_') // Replace spaces with underscores } - -export function replaceCandidateInContent( - content: string, - replacement: string, - startByte: number, - endByte: number, -) { - return stringByteSlice(content, 0, startByte) + replacement + stringByteSlice(content, endByte) -} diff --git a/packages/@tailwindcss-upgrade/src/template/migrate.ts b/packages/@tailwindcss-upgrade/src/template/migrate.ts index 50f2ca60e..3e4c498d5 100644 --- a/packages/@tailwindcss-upgrade/src/template/migrate.ts +++ b/packages/@tailwindcss-upgrade/src/template/migrate.ts @@ -2,7 +2,7 @@ import fs from 'node:fs/promises' import path, { extname } from 'node:path' import type { Config } from 'tailwindcss' import type { DesignSystem } from '../../../tailwindcss/src/design-system' -import { extractRawCandidates, replaceCandidateInContent } from './candidates' +import { extractRawCandidates } from './candidates' import { arbitraryValueToBareValue } from './codemods/arbitrary-value-to-bare-value' import { automaticVarInjection } from './codemods/automatic-var-injection' import { bgGradient } from './codemods/bg-gradient' @@ -10,6 +10,7 @@ import { important } from './codemods/important' import { prefix } from './codemods/prefix' import { simpleLegacyClasses } from './codemods/simple-legacy-classes' import { variantOrder } from './codemods/variant-order' +import { spliceChangesIntoString, type StringChange } from './splice-changes-into-string' export type Migration = ( designSystem: DesignSystem, @@ -46,19 +47,23 @@ export default async function migrateContents( ): Promise { let candidates = await extractRawCandidates(contents, extension) - // Sort candidates by starting position desc - candidates.sort((a, z) => z.start - a.start) + let changes: StringChange[] = [] - let output = contents for (let { rawCandidate, start, end } of candidates) { let migratedCandidate = migrateCandidate(designSystem, userConfig, rawCandidate) - if (migratedCandidate !== rawCandidate) { - output = replaceCandidateInContent(output, migratedCandidate, start, end) + if (migratedCandidate === rawCandidate) { + continue } + + changes.push({ + start, + end, + replacement: migratedCandidate, + }) } - return output + return spliceChangesIntoString(contents, changes) } export async function migrate(designSystem: DesignSystem, userConfig: Config, file: string) { diff --git a/packages/@tailwindcss-upgrade/src/template/splice-changes-into-string.ts b/packages/@tailwindcss-upgrade/src/template/splice-changes-into-string.ts new file mode 100644 index 000000000..e1e6f1685 --- /dev/null +++ b/packages/@tailwindcss-upgrade/src/template/splice-changes-into-string.ts @@ -0,0 +1,42 @@ +export interface StringChange { + start: number + end: number + replacement: string +} + +/** + * Apply the changes to the string such that a change in the length + * of the string does not break the indexes of the subsequent changes. + */ +export function spliceChangesIntoString(str: string, changes: StringChange[]) { + // If there are no changes, return the original string + if (!changes[0]) return str + + // Sort all changes in order to make it easier to apply them + changes.sort((a, b) => { + return a.end - b.end || a.start - b.start + }) + + // Append original string between each chunk, and then the chunk itself + // This is sort of a String Builder pattern, thus creating less memory pressure + let result = '' + + let previous = changes[0] + + result += str.slice(0, previous.start) + result += previous.replacement + + for (let i = 1; i < changes.length; ++i) { + let change = changes[i] + + result += str.slice(previous.end, change.start) + result += change.replacement + + previous = change + } + + // Add leftover string from last chunk to end + result += str.slice(previous.end) + + return result +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9cdd4f1c6..a1c7cabab 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -306,9 +306,6 @@ importers: prettier: specifier: ^3.3.3 version: 3.3.3 - string-byte-slice: - specifier: ^3.0.0 - version: 3.0.0 tailwindcss: specifier: workspace:^ version: link:../tailwindcss @@ -1009,7 +1006,6 @@ packages: '@parcel/watcher-darwin-arm64@2.4.2-alpha.0': resolution: {integrity: sha512-2xH4Ve7OKjIh+4YRfTN3HGJa2W8KTPLOALHZj5fxcbTPwaVxdpIRItDrcikUx2u3AzGAFme7F+AZZXHnf0F15Q==} engines: {node: '>= 10.0.0'} - cpu: [arm64] os: [darwin] '@parcel/watcher-darwin-x64@2.4.1': @@ -1021,7 +1017,6 @@ packages: '@parcel/watcher-darwin-x64@2.4.2-alpha.0': resolution: {integrity: sha512-xtjmXUH4YZVah5+7Q0nb+fpRP5qZn9cFfuPuZ4k77UfUGVwhacgZyIRQgIOwMP3GkgW4TsrKQaw1KIe7L1ZqcQ==} engines: {node: '>= 10.0.0'} - cpu: [x64] os: [darwin] '@parcel/watcher-freebsd-x64@2.4.1': @@ -1045,7 +1040,6 @@ packages: '@parcel/watcher-linux-arm64-glibc@2.4.2-alpha.0': resolution: {integrity: sha512-vIIOcZf+fgsRReIK3Fw0WINvGo9UwiXfisnqYRzfpNByRZvkEPkGTIVe8iiDp72NhPTVmwIvBqM6yKDzIaw8GQ==} engines: {node: '>= 10.0.0'} - cpu: [arm64] os: [linux] '@parcel/watcher-linux-arm64-musl@2.4.1': @@ -1057,7 +1051,6 @@ packages: '@parcel/watcher-linux-arm64-musl@2.4.2-alpha.0': resolution: {integrity: sha512-gXqEAoLG9bBCbQNUgqjSOxHcjpmCZmYT9M8UvrdTMgMYgXgiWcR8igKlPRd40mCIRZSkMpN2ScSy2WjQ0bQZnQ==} engines: {node: '>= 10.0.0'} - cpu: [arm64] os: [linux] '@parcel/watcher-linux-x64-glibc@2.4.1': @@ -1069,7 +1062,6 @@ packages: '@parcel/watcher-linux-x64-glibc@2.4.2-alpha.0': resolution: {integrity: sha512-/WJJ3Y46ubwQW+Z+mzpzK3pvqn/AT7MA63NB0+k9GTLNxJQZNREensMtpJ/FJ+LVIiraEHTY22KQrsx9+DeNbw==} engines: {node: '>= 10.0.0'} - cpu: [x64] os: [linux] '@parcel/watcher-linux-x64-musl@2.4.1': @@ -1081,7 +1073,6 @@ packages: '@parcel/watcher-linux-x64-musl@2.4.2-alpha.0': resolution: {integrity: sha512-1dz4fTM5HaANk3RSRmdhALT+bNqTHawVDL1D77HwV/FuF/kSjlM3rGrJuFaCKwQ5E8CInHCcobqMN8Jh8LYaRg==} engines: {node: '>= 10.0.0'} - cpu: [x64] os: [linux] '@parcel/watcher-win32-arm64@2.4.1': @@ -1105,7 +1096,6 @@ packages: '@parcel/watcher-win32-x64@2.4.2-alpha.0': resolution: {integrity: sha512-U2abMKF7JUiIxQkos19AvTLFcnl2Xn8yIW1kzu+7B0Lux4Gkuu/BUDBroaM1s6+hwgK63NOLq9itX2Y3GwUThg==} engines: {node: '>= 10.0.0'} - cpu: [x64] os: [win32] '@parcel/watcher@2.4.1': @@ -1441,7 +1431,6 @@ packages: bun@1.1.29: resolution: {integrity: sha512-SKhpyKNZtgxrVel9ec9xon3LDv8mgpiuFhARgcJo1YIbggY2PBrKHRNiwQ6Qlb+x3ivmRurfuwWgwGexjpgBRg==} - cpu: [arm64, x64] os: [darwin, linux, win32] hasBin: true @@ -2210,13 +2199,11 @@ packages: lightningcss-darwin-arm64@1.26.0: resolution: {integrity: sha512-n4TIvHO1NY1ondKFYpL2ZX0bcC2y6yjXMD6JfyizgR8BCFNEeArINDzEaeqlfX9bXz73Bpz/Ow0nu+1qiDrBKg==} engines: {node: '>= 12.0.0'} - cpu: [arm64] os: [darwin] lightningcss-darwin-x64@1.26.0: resolution: {integrity: sha512-Rf9HuHIDi1R6/zgBkJh25SiJHF+dm9axUZW/0UoYCW1/8HV0gMI0blARhH4z+REmWiU1yYT/KyNF3h7tHyRXUg==} engines: {node: '>= 12.0.0'} - cpu: [x64] os: [darwin] lightningcss-freebsd-x64@1.26.0: @@ -2234,25 +2221,21 @@ packages: lightningcss-linux-arm64-gnu@1.26.0: resolution: {integrity: sha512-iJmZM7fUyVjH+POtdiCtExG+67TtPUTer7K/5A8DIfmPfrmeGvzfRyBltGhQz13Wi15K1lf2cPYoRaRh6vcwNA==} engines: {node: '>= 12.0.0'} - cpu: [arm64] os: [linux] lightningcss-linux-arm64-musl@1.26.0: resolution: {integrity: sha512-XxoEL++tTkyuvu+wq/QS8bwyTXZv2y5XYCMcWL45b8XwkiS8eEEEej9BkMGSRwxa5J4K+LDeIhLrS23CpQyfig==} engines: {node: '>= 12.0.0'} - cpu: [arm64] os: [linux] lightningcss-linux-x64-gnu@1.26.0: resolution: {integrity: sha512-1dkTfZQAYLj8MUSkd6L/+TWTG8V6Kfrzfa0T1fSlXCXQHrt1HC1/UepXHtKHDt/9yFwyoeayivxXAsApVxn6zA==} engines: {node: '>= 12.0.0'} - cpu: [x64] os: [linux] lightningcss-linux-x64-musl@1.26.0: resolution: {integrity: sha512-yX3Rk9m00JGCUzuUhFEojY+jf/6zHs3XU8S8Vk+FRbnr4St7cjyMXdNjuA2LjiT8e7j8xHRCH8hyZ4H/btRE4A==} engines: {node: '>= 12.0.0'} - cpu: [x64] os: [linux] lightningcss-win32-arm64-msvc@1.26.0: @@ -2264,7 +2247,6 @@ packages: lightningcss-win32-x64-msvc@1.26.0: resolution: {integrity: sha512-pYS3EyGP3JRhfqEFYmfFDiZ9/pVNfy8jVIYtrx9TVNusVyDK3gpW1w/rbvroQ4bDJi7grdUtyrYU6V2xkY/bBw==} engines: {node: '>= 12.0.0'} - cpu: [x64] os: [win32] lightningcss@1.26.0: @@ -2792,10 +2774,6 @@ packages: resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==} engines: {node: '>=10.0.0'} - string-byte-slice@3.0.0: - resolution: {integrity: sha512-KqTTvThKPDgBPr9jI2cOdO04tJ+upcADk4j4zmcBNmG6Bqstsq1x1Z3xvJAPqRQgPE8yocXNLVZuCfYlv4+PTg==} - engines: {node: '>=18.18.0'} - string-width@4.2.3: resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} engines: {node: '>=8'} @@ -4403,7 +4381,7 @@ snapshots: eslint: 8.57.0 eslint-import-resolver-node: 0.3.9 eslint-import-resolver-typescript: 3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.57.0))(eslint@8.57.0) - eslint-plugin-import: 2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.5.4))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0) + eslint-plugin-import: 2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.5.4))(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0) eslint-plugin-jsx-a11y: 6.9.0(eslint@8.57.0) eslint-plugin-react: 7.35.0(eslint@8.57.0) eslint-plugin-react-hooks: 4.6.2(eslint@8.57.0) @@ -4427,7 +4405,7 @@ snapshots: enhanced-resolve: 5.17.1 eslint: 8.57.0 eslint-module-utils: 2.8.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0) - eslint-plugin-import: 2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.5.4))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0) + eslint-plugin-import: 2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.5.4))(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0) fast-glob: 3.3.2 get-tsconfig: 4.7.6 is-core-module: 2.15.0 @@ -4449,7 +4427,7 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-plugin-import@2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.5.4))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0): + eslint-plugin-import@2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.5.4))(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0): dependencies: array-includes: 3.1.8 array.prototype.findlastindex: 1.2.5 @@ -5566,8 +5544,6 @@ snapshots: streamsearch@1.1.0: {} - string-byte-slice@3.0.0: {} - string-width@4.2.3: dependencies: emoji-regex: 8.0.0