Skip to content

js_parser: delete lexer::range_of_identifier, call Source::range_of_identifier #48100

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
robobun wants to merge 1 commit into
base: main
Choose a base branch
from
Open

js_parser: delete lexer::range_of_identifier, call Source::range_of_identifier #48100

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions src/bundler/LinkerContext.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ use bun_ast::{
// re-export, so use that here.
use crate::Index;
use bun_ast::{E, G, S};
use bun_js_parser::lexer as lex;
use bun_js_parser::parse::parse_entry::{ModuleSyntax, parse_only_text};
use bun_js_printer::{self as js_printer, renamer};

Expand Down Expand Up @@ -4465,7 +4464,7 @@ impl<'a> LinkerContext<'a> {
}
MatchImportKind::Cycle => {
let source = self.get_source(source_index);
let r = lex::range_of_identifier(source, named_import.alias_loc);
let r = source.range_of_identifier(named_import.alias_loc);
// SAFETY: arena `*const [u8]` valid for the link pass.
let alias = named_import
.alias
Expand All @@ -4489,7 +4488,7 @@ impl<'a> LinkerContext<'a> {
}
MatchImportKind::Ambiguous => {
let source = self.get_source(source_index);
let r = lex::range_of_identifier(source, named_import.alias_loc);
let r = source.range_of_identifier(named_import.alias_loc);

// TODO: log locations of the ambiguous exports

Expand Down
61 changes: 1 addition & 60 deletions src/js_parser/lexer.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1872,7 +1872,7 @@ impl<'a> Lexer<'a> {
if !self.fn_or_arrow_start_loc.is_empty() {
notes[0] = bun_ast::range_data(
Some(self.source),
range_of_identifier(self.source, self.fn_or_arrow_start_loc),
self.source.range_of_identifier(self.fn_or_arrow_start_loc),
b"Consider adding the \"async\" keyword here",
);
}
Expand Down Expand Up @@ -3348,65 +3348,6 @@ pub fn is_identifier_continue(codepoint: i32) -> bool {

pub use bun_core::identifier::{is_identifier, is_identifier_utf16};

pub fn range_of_identifier(source: &Source, loc: Loc) -> Range {
let contents = &source.contents;
if loc.start == -1 || usize::try_from(loc.start).expect("int cast") >= contents.len() {
return Range::NONE;
}

let iter = CodepointIterator::init(&contents[loc.to_usize()..]);
let mut cursor = strings::Cursor::default();

let mut r = Range { loc, len: 0 };
if iter.bytes.is_empty() {
return r;
}
let text = iter.bytes;
let end = u32::try_from(text.len()).expect("int cast");

if !iter.next(&mut cursor) {
return r;
}

// Handle private names
if cursor.c == 0x23 {
if !iter.next(&mut cursor) {
r.len = 1;
return r;
}
}

if is_identifier_start(cursor.c) || cursor.c == 0x5C {
while iter.next(&mut cursor) {
if cursor.c == 0x5C {
// Search for the end of the identifier

// Skip over bracketed unicode escapes such as "\u{10000}"
if cursor.i + 2 < end
&& text[cursor.i as usize + 1] == b'u'
&& text[cursor.i as usize + 2] == b'{'
{
cursor.i += 2;
while cursor.i < end {
if text[cursor.i as usize] == b'}' {
cursor.i += 1;
break;
}
cursor.i += 1;
}
}
} else if !is_identifier_continue(cursor.c) {
r.len = i32::try_from(cursor.i).expect("int cast");
return r;
}
}

r.len = i32::try_from(cursor.i).expect("int cast");
}

r
}

// PERF: force-inline — sole call site is the identifier arm of `next()`, the
// hottest token by frequency. It's tiny, so it belongs *inside* `next()`'s
// body (a call + ret per identifier would dominate it).
Expand Down
30 changes: 13 additions & 17 deletions src/js_parser/p.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -953,7 +953,7 @@ impl<'a, const TYPESCRIPT: bool, const SCAN_ONLY: bool> P<'a, TYPESCRIPT, SCAN_O
let mut shape_buf = BumpVec::new_in(self.arena);
let shape = self.extract_dynamic_specifier_shape(arg, &mut shape_buf)?;
if !self.options.allow_unresolved.allows(shape) {
let r = js_lexer::range_of_identifier(self.source, loc);
let r = self.source.range_of_identifier(loc);
if !shape.is_empty() {
// Print a human-readable shape: replace \x00 with *
let display = self.arena.alloc_slice_copy(shape);
Expand Down Expand Up @@ -1513,7 +1513,7 @@ impl<'a, const TYPESCRIPT: bool, const SCAN_ONLY: bool> P<'a, TYPESCRIPT, SCAN_O

if self.options.warn_about_unbundled_modules {
// Use a debug log so people can see this if they want to
let r = js_lexer::range_of_identifier(self.source, state.loc);
let r = self.source.range_of_identifier(state.loc);
self.log()
.add_range_debug(
Some(self.source),
Expand Down Expand Up @@ -1547,7 +1547,7 @@ impl<'a, const TYPESCRIPT: bool, const SCAN_ONLY: bool> P<'a, TYPESCRIPT, SCAN_O

if self.options.warn_about_unbundled_modules {
// Use a debug log so people can see this if they want to
let r = js_lexer::range_of_identifier(self.source, arg.loc);
let r = self.source.range_of_identifier(arg.loc);
self.log()
.add_range_debug(
Some(self.source),
Expand Down Expand Up @@ -1963,13 +1963,13 @@ impl<'a, const TYPESCRIPT: bool, const SCAN_ONLY: bool> P<'a, TYPESCRIPT, SCAN_O
),
location: bun_ast::Location::init_or_null(
Some(self.source),
js_lexer::range_of_identifier(self.source, name.alias_loc),
self.source.range_of_identifier(name.alias_loc),
),
..Default::default()
}]);
self.log().add_range_error_fmt_with_notes(
Some(self.source),
js_lexer::range_of_identifier(self.source, loc),
self.source.range_of_identifier(loc),
notes,
format_args!(
"Multiple exports with the same name \"{}\"",
Expand Down Expand Up @@ -2192,7 +2192,7 @@ impl<'a, const TYPESCRIPT: bool, const SCAN_ONLY: bool> P<'a, TYPESCRIPT, SCAN_O
&& (opts.assign_target() != js_ast::AssignTarget::None || opts.is_delete_target())
&& self.symbols[ref_.inner_index() as usize].kind == js_ast::symbol::Kind::Import
{
let r = js_lexer::range_of_identifier(self.source, loc);
let r = self.source.range_of_identifier(loc);
// SAFETY: original_name is an arena-owned slice valid for 'a.
let original = self.symbols[ref_.inner_index() as usize]
.original_name
Expand Down Expand Up @@ -3640,8 +3640,7 @@ impl<'a, const TYPESCRIPT: bool, const SCAN_ONLY: bool> P<'a, TYPESCRIPT, SCAN_O
!= js_ast::symbol::Kind::HoistedFunction
{
if !is_sloppy_mode_block_level_fn_stmt {
let r =
js_lexer::range_of_identifier(self.source, value.loc);
let r = self.source.range_of_identifier(value.loc);
let mut msg = Vec::<u8>::new();
let _ = write!(
&mut msg,
Expand All @@ -3656,10 +3655,7 @@ impl<'a, const TYPESCRIPT: bool, const SCAN_ONLY: bool> P<'a, TYPESCRIPT, SCAN_O
)]);
self.log().add_range_error_fmt_with_notes(
Some(self.source),
js_lexer::range_of_identifier(
self.source,
member_in_scope.loc,
),
self.source.range_of_identifier(member_in_scope.loc),
notes,
format_args!(
"{} has already been declared",
Expand Down Expand Up @@ -4718,7 +4714,7 @@ impl<'a, const TYPESCRIPT: bool, const SCAN_ONLY: bool> P<'a, TYPESCRIPT, SCAN_O
if decl.value.is_none() {
match &decl.binding.data {
js_ast::b::B::BIdentifier(ident) => {
let r = js_lexer::range_of_identifier(self.source, decl.binding.loc);
let r = self.source.range_of_identifier(decl.binding.loc);
let ident_ref = ident.r#ref;
// SAFETY: original_name is an arena-owned slice valid for 'a.
let name = self.symbols[ident_ref.inner_index() as usize]
Expand Down Expand Up @@ -5048,7 +5044,7 @@ impl<'a, const TYPESCRIPT: bool, const SCAN_ONLY: bool> P<'a, TYPESCRIPT, SCAN_O
{
self.mark_strict_mode_feature(
StrictModeFeature::ReservedWord,
js_lexer::range_of_identifier(self.source, loc),
self.source.range_of_identifier(loc),
name,
)?;
}
Expand Down Expand Up @@ -5148,14 +5144,14 @@ impl<'a, const TYPESCRIPT: bool, const SCAN_ONLY: bool> P<'a, TYPESCRIPT, SCAN_O
if func.flags.contains(Flags::Function::IsAsync) && original_name == b"await" {
self.log().add_range_error(
Some(self.source),
js_lexer::range_of_identifier(self.source, name.loc),
self.source.range_of_identifier(name.loc),
b"An async function cannot be named \"await\"",
);
} else if func.flags.contains(Flags::Function::IsGenerator) && original_name == b"yield"
{
self.log().add_range_error(
Some(self.source),
js_lexer::range_of_identifier(self.source, name.loc),
self.source.range_of_identifier(name.loc),
b"An generator function expression cannot be named \"yield\"",
);
}
Expand Down Expand Up @@ -8113,7 +8109,7 @@ impl<'a, const TYPESCRIPT: bool, const SCAN_ONLY: bool> P<'a, TYPESCRIPT, SCAN_O
_scope = scope.parent;
}

let r = js_lexer::range_of_identifier(self.source, loc);
let r = self.source.range_of_identifier(loc);
self.log().add_range_error_fmt(
Some(self.source),
r,
Expand Down
13 changes: 8 additions & 5 deletions src/js_parser/parse/parse_import_export.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
use crate::Error;
use crate::lexer::{self as js_lexer, T};
use crate::lexer::T;
use crate::p::P;
use crate::parser::{ExportClauseResult, ImportClause, is_eval_or_arguments};
use bun_alloc::ArenaVecExt as _;
Expand All @@ -18,7 +18,10 @@ impl<'a, const TYPESCRIPT: bool, const SCAN_ONLY: bool> P<'a, TYPESCRIPT, SCAN_O
let p = self;
// Parse an "import.meta" expression
if p.lexer.token == T::TDot {
p.esm_import_keyword = js_lexer::range_of_identifier(p.source, loc);
p.esm_import_keyword = bun_ast::Range {
loc,
len: b"import".len() as i32,
};
p.lexer.next()?;
if p.lexer.is_contextual_keyword(b"meta") {
p.lexer.next()?;
Expand All @@ -30,7 +33,7 @@ impl<'a, const TYPESCRIPT: bool, const SCAN_ONLY: bool> P<'a, TYPESCRIPT, SCAN_O
}

if level.gt(Level::Call) {
let r = js_lexer::range_of_identifier(p.source, loc);
let r = p.source.range_of_identifier(loc);
p.log().add_range_error(
Some(p.source),
r,
Expand Down Expand Up @@ -238,7 +241,7 @@ impl<'a, const TYPESCRIPT: bool, const SCAN_ONLY: bool> P<'a, TYPESCRIPT, SCAN_O

// Reject forbidden names
if is_eval_or_arguments(original_name) {
let r = js_lexer::range_of_identifier(p.source, name.loc);
let r = p.source.range_of_identifier(name.loc);
p.log().add_range_error_fmt(
Some(p.source),
r,
Expand Down Expand Up @@ -452,7 +455,7 @@ impl<'a, const TYPESCRIPT: bool, const SCAN_ONLY: bool> P<'a, TYPESCRIPT, SCAN_O
// Throw an error here if we found a keyword earlier and this isn't an
// "export from" statement after all
if first_non_identifier_loc.start != 0 && !p.lexer.is_contextual_keyword(b"from") {
let r = js_lexer::range_of_identifier(p.source, first_non_identifier_loc);
let r = p.source.range_of_identifier(first_non_identifier_loc);
p.lexer.add_range_error(
r,
format_args!(
Expand Down
17 changes: 7 additions & 10 deletions src/js_parser/parse/parse_property.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ impl<'a, const TYPESCRIPT: bool, const SCAN_ONLY: bool> P<'a, TYPESCRIPT, SCAN_O
match kind {
PropertyKind::Get => {
if args.len() > 0 {
let r = js_lexer::range_of_identifier(p.source, args[0].binding.loc);
let r = p.source.range_of_identifier(args[0].binding.loc);
let key_name = p.key_name_for_error(key);
p.log().add_range_error_fmt(
Some(p.source),
Expand All @@ -150,16 +150,13 @@ impl<'a, const TYPESCRIPT: bool, const SCAN_ONLY: bool> P<'a, TYPESCRIPT, SCAN_O
}
PropertyKind::Set => {
if args.len() != 1 {
let mut r = js_lexer::range_of_identifier(
p.source,
if args.len() > 0 {
args[0].binding.loc
} else {
loc
},
);
let mut r = p.source.range_of_identifier(if args.len() > 0 {
args[0].binding.loc
} else {
loc
});
if args.len() > 1 {
r = js_lexer::range_of_identifier(p.source, args[1].binding.loc);
r = p.source.range_of_identifier(args[1].binding.loc);
}
let key_name = p.key_name_for_error(key);
p.log().add_range_error_fmt(
Expand Down
10 changes: 5 additions & 5 deletions src/js_parser/parse/parse_stmt.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1092,7 +1092,7 @@ impl<'a, const TYPESCRIPT: bool, const SCAN_ONLY: bool> P<'a, TYPESCRIPT, SCAN_O
// building an S.ExportDefault that the visit and print
// passes don't support.
_ => {
let r = js_lexer::range_of_identifier(p.source, stmt.loc);
let r = p.source.range_of_identifier(stmt.loc);
p.log().add_range_error_fmt(
Some(p.source),
r,
Expand Down Expand Up @@ -1182,7 +1182,7 @@ impl<'a, const TYPESCRIPT: bool, const SCAN_ONLY: bool> P<'a, TYPESCRIPT, SCAN_O
&& p.lexer.has_newline_before
&& matches!(expr.data, js_ast::ExprData::EIdentifier(_))
{
let r = js_lexer::range_of_identifier(p.source, expr.loc);
let r = p.source.range_of_identifier(expr.loc);
p.log()
.add_range_error(Some(p.source), r, b"Unexpected \"abstract\"");
return Err(crate::Error::SyntaxError);
Expand Down Expand Up @@ -1835,7 +1835,7 @@ impl<'a, const TYPESCRIPT: bool, const SCAN_ONLY: bool> P<'a, TYPESCRIPT, SCAN_O
// "interface \n Foo {}"
// "export interface \n Foo {}"
if opts.is_export {
let r = js_lexer::range_of_identifier(p.source, loc);
let r = p.source.range_of_identifier(loc);
p.log()
.add_range_error(Some(p.source), r, b"Unexpected \"interface\"");
return Err(crate::Error::SyntaxError);
Expand All @@ -1848,7 +1848,7 @@ impl<'a, const TYPESCRIPT: bool, const SCAN_ONLY: bool> P<'a, TYPESCRIPT, SCAN_O
return Ok(Some(p.parse_class_stmt(p.lexer.loc(), opts)?));
}
if opts.ts_decorators.is_some() {
let r = js_lexer::range_of_identifier(p.source, loc);
let r = p.source.range_of_identifier(loc);
p.log()
.add_range_error(Some(p.source), r, b"Unexpected \"abstract\"");
return Err(crate::Error::SyntaxError);
Expand All @@ -1870,7 +1870,7 @@ impl<'a, const TYPESCRIPT: bool, const SCAN_ONLY: bool> P<'a, TYPESCRIPT, SCAN_O
js_lexer::TypescriptStmtKeyword::TsStmtDeclare => {
if p.lexer.has_newline_before {
if opts.ts_decorators.is_some() {
let r = js_lexer::range_of_identifier(p.source, loc);
let r = p.source.range_of_identifier(loc);
p.log()
.add_range_error(Some(p.source), r, b"Unexpected \"declare\"");
return Err(crate::Error::SyntaxError);
Expand Down
3 changes: 1 addition & 2 deletions src/js_parser/scan/scan_symbols.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
use crate::js_lexer;
use crate::p::P;
use crate::parser::FindSymbolResult;
use bun_ast as js_ast;
Expand Down Expand Up @@ -43,7 +42,7 @@ impl<'a, const TYPESCRIPT: bool, const SCAN_ONLY: bool> P<'a, TYPESCRIPT, SCAN_O

// Forbid referencing "arguments" inside class bodies
if scope.forbid_arguments && !did_forbid_arguments && name == b"arguments" {
let r = js_lexer::range_of_identifier(self.source, loc);
let r = self.source.range_of_identifier(loc);
self.log().add_range_error_fmt(
Some(self.source),
r,
Expand Down
7 changes: 3 additions & 4 deletions src/js_parser/visit/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ pub mod visit_binary;
pub(crate) mod visit_expr;
pub(crate) mod visit_stmt;

use crate::lexer as js_lexer;
use crate::p::{LowerUsingDeclarationsContext, P};
use crate::parser::{
ExprIn, FnOnlyDataVisit, FnOrArrowDataVisit, ImportItemForNamespaceMap, PrependTempRefsOpts,
Expand Down Expand Up @@ -105,7 +104,7 @@ impl<'a, const TYPESCRIPT: bool, const SCAN_ONLY: bool> P<'a, TYPESCRIPT, SCAN_O
if is_eval_or_arguments(symbol_name) {
self.mark_strict_mode_feature(
StrictModeFeature::EvalOrArguments,
js_lexer::range_of_identifier(self.source, name.loc),
self.source.range_of_identifier(name.loc),
symbol_name,
)
.expect("unreachable");
Expand Down Expand Up @@ -820,7 +819,7 @@ impl<'a, const TYPESCRIPT: bool, const SCAN_ONLY: bool> P<'a, TYPESCRIPT, SCAN_O
if is_eval_or_arguments(name) {
self.mark_strict_mode_feature(
StrictModeFeature::EvalOrArguments,
js_lexer::range_of_identifier(self.source, binding.loc),
self.source.range_of_identifier(binding.loc),
name,
)
.expect("unreachable");
Expand All @@ -829,7 +828,7 @@ impl<'a, const TYPESCRIPT: bool, const SCAN_ONLY: bool> P<'a, TYPESCRIPT, SCAN_O
if dup.get_or_put_contains(name) {
self.log().add_range_error_fmt(
Some(self.source),
js_lexer::range_of_identifier(self.source, binding.loc),
self.source.range_of_identifier(binding.loc),
format_args!(
"\"{}\" cannot be bound multiple times in the same parameter list",
bstr::BStr::new(name)
Expand Down
Loading
Loading