From 3309d3f635b53a3c7e21a4d58963d32aea689cf2 Mon Sep 17 00:00:00 2001 From: Robin Malfait Date: Thu, 27 Apr 2023 23:34:08 +0200 Subject: [PATCH] [oxide] Expose experimental Rust parser setup (#11116) * make `sequential` and `parallel` version of a new (tmp) `parse_candidate_strings` * use bitmasks for the strategy Only sending a number over the wire instead of a serialized objects. * use cleaner match syntax --- oxide/crates/core/src/lib.rs | 105 ++++++++++++++++++++++++++++++++--- oxide/crates/node/src/lib.rs | 39 ++++++++++--- 2 files changed, 128 insertions(+), 16 deletions(-) diff --git a/oxide/crates/core/src/lib.rs b/oxide/crates/core/src/lib.rs index 84713e61b..9dd88437d 100644 --- a/oxide/crates/core/src/lib.rs +++ b/oxide/crates/core/src/lib.rs @@ -1,4 +1,5 @@ use crate::parser::Extractor; +use fxhash::FxHashSet; use rayon::prelude::*; use std::path::PathBuf; use tracing::event; @@ -11,14 +12,7 @@ pub mod parser; pub mod utility; pub mod variant; -#[derive(Debug, Clone)] -pub struct ChangedContent { - pub file: Option, - pub content: Option, - pub extension: String, -} - -pub fn parse_candidate_strings_from_files(changed_content: Vec) -> Vec { +fn init_tracing() { if matches!(std::env::var("DEBUG"), Ok(value) if value.eq("*") || value.eq("1") || value.eq("true") || value.contains("tailwind")) { tracing_subscriber::fmt() @@ -27,10 +21,63 @@ pub fn parse_candidate_strings_from_files(changed_content: Vec) .compact() .init(); } +} +#[derive(Debug, Clone)] +pub struct ChangedContent { + pub file: Option, + pub content: Option, + pub extension: String, +} + +#[derive(Debug)] +pub enum IO { + Sequential = 0b0001, + Parallel = 0b0010, +} + +impl From for IO { + fn from(item: u8) -> Self { + match item & 0b0011 { + 0b0001 => IO::Sequential, + 0b0010 => IO::Parallel, + _ => unimplemented!("Unknown 'IO' strategy"), + } + } +} + +#[derive(Debug)] +pub enum Parsing { + Sequential = 0b0100, + Parallel = 0b1000, +} + +impl From for Parsing { + fn from(item: u8) -> Self { + match item & 0b1100 { + 0b0100 => Parsing::Sequential, + 0b1000 => Parsing::Parallel, + _ => unimplemented!("Unknown 'Parsing' strategy"), + } + } +} + +pub fn parse_candidate_strings_from_files(changed_content: Vec) -> Vec { + init_tracing(); parse_all_blobs(read_all_files(changed_content)) } +pub fn parse_candidate_strings(input: Vec, options: u8) -> Vec { + init_tracing(); + + match (IO::from(options), Parsing::from(options)) { + (IO::Sequential, Parsing::Sequential) => parse_all_blobs_sync(read_all_files_sync(input)), + (IO::Sequential, Parsing::Parallel) => parse_all_blobs_sync(read_all_files(input)), + (IO::Parallel, Parsing::Sequential) => parse_all_blobs(read_all_files_sync(input)), + (IO::Parallel, Parsing::Parallel) => parse_all_blobs(read_all_files(input)), + } +} + #[tracing::instrument(skip(changed_content))] fn read_all_files(changed_content: Vec) -> Vec> { event!( @@ -49,6 +96,24 @@ fn read_all_files(changed_content: Vec) -> Vec> { .collect() } +#[tracing::instrument(skip(changed_content))] +fn read_all_files_sync(changed_content: Vec) -> Vec> { + event!( + tracing::Level::INFO, + "Reading {:?} file(s)", + changed_content.len() + ); + + changed_content + .into_iter() + .map(|c| match (c.file, c.content) { + (Some(file), None) => std::fs::read(file).unwrap(), + (None, Some(content)) => content.into_bytes(), + _ => Default::default(), + }) + .collect() +} + #[tracing::instrument(skip(blobs))] fn parse_all_blobs(blobs: Vec>) -> Vec { let input: Vec<_> = blobs.iter().map(|blob| &blob[..]).collect(); @@ -72,3 +137,27 @@ fn parse_all_blobs(blobs: Vec>) -> Vec { result.sort(); result } + +#[tracing::instrument(skip(blobs))] +fn parse_all_blobs_sync(blobs: Vec>) -> Vec { + let input: Vec<_> = blobs.iter().map(|blob| &blob[..]).collect(); + let input = &input[..]; + + let mut result: Vec = input + .iter() + .map(|input| Extractor::unique(input, Default::default())) + .fold(FxHashSet::default(), |mut a, b| { + a.extend(b); + a + }) + .into_iter() + .map(|s| { + // SAFETY: When we parsed the candidates, we already guaranteed that the byte slices + // are valid, therefore we don't have to re-check here when we want to convert it back + // to a string. + unsafe { String::from_utf8_unchecked(s.to_vec()) } + }) + .collect(); + result.sort(); + result +} diff --git a/oxide/crates/node/src/lib.rs b/oxide/crates/node/src/lib.rs index 20961adcb..f8b927db5 100644 --- a/oxide/crates/node/src/lib.rs +++ b/oxide/crates/node/src/lib.rs @@ -1,3 +1,4 @@ +use napi::bindgen_prelude::ToNapiValue; use std::path::PathBuf; #[macro_use] @@ -11,16 +12,38 @@ pub struct ChangedContent { pub extension: String, } +impl From for tailwindcss_core::ChangedContent { + fn from(changed_content: ChangedContent) -> Self { + tailwindcss_core::ChangedContent { + file: changed_content.file.map(PathBuf::from), + content: changed_content.content, + extension: changed_content.extension, + } + } +} + #[napi] pub fn parse_candidate_strings_from_files(changed_content: Vec) -> Vec { tailwindcss_core::parse_candidate_strings_from_files( - changed_content - .into_iter() - .map(|changed_content| tailwindcss_core::ChangedContent { - file: changed_content.file.map(PathBuf::from), - content: changed_content.content, - extension: changed_content.extension, - }) - .collect(), + changed_content.into_iter().map(Into::into).collect(), ) } + +#[derive(Debug)] +#[napi] +pub enum IO { + Sequential = 0b0001, + Parallel = 0b0010, +} + +#[derive(Debug)] +#[napi] +pub enum Parsing { + Sequential = 0b0100, + Parallel = 0b1000, +} + +#[napi] +pub fn parse_candidate_strings(input: Vec, strategy: u8) -> Vec { + tailwindcss_core::parse_candidate_strings(input.into_iter().map(Into::into).collect(), strategy) +}