From bfbd4dc594e6008e119dee5232734c9e94436023 Mon Sep 17 00:00:00 2001 From: Toddr Bot Date: Mon, 27 Apr 2026 08:26:24 +0000 Subject: [PATCH 1/2] test: add failing tests for ellipsis '...' token type (issue #41) The Perl 5.12 ellipsis statement '...' (yada yada) currently parses as PPI::Token::Operator. Add $TODO-marked tests that verify it should become PPI::Token::Ellipsis and its containing statement should be PPI::Statement::Break. Co-Authored-By: Claude Opus 4.6 --- t/ppi_token_ellipsis.t | 94 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 t/ppi_token_ellipsis.t diff --git a/t/ppi_token_ellipsis.t b/t/ppi_token_ellipsis.t new file mode 100644 index 00000000..74796106 --- /dev/null +++ b/t/ppi_token_ellipsis.t @@ -0,0 +1,94 @@ +#!/usr/bin/perl + +# Unit testing for PPI::Token::Ellipsis + +use lib 't/lib'; +use PPI::Test::pragmas; +use Test::More tests => 38 + ($ENV{AUTHOR_TESTING} ? 1 : 0); + +use PPI (); +use Helper 'safe_new'; + +our $TODO; + +ELLIPSIS_TOKEN_TYPE: { + my $doc = safe_new \'...;'; + + local $TODO = "ellipsis token type not yet implemented"; + + my $tokens = $doc->find( sub { $_[1]->isa('PPI::Token') and $_[1]->content eq '...' } ); + is( ref $tokens, 'ARRAY', "'...' token found" ); + is( @$tokens, 1, "'...' found exactly once" ); + isa_ok( $tokens->[0], 'PPI::Token::Ellipsis', "'...' token" ); + ok( !$tokens->[0]->isa('PPI::Token::Operator'), "'...' is not a Token::Operator" ); +} + +ELLIPSIS_IN_SUB: { + my $doc = safe_new \'sub foo { ... }'; + + local $TODO = "ellipsis token type not yet implemented"; + + my $tokens = $doc->find( sub { $_[1]->isa('PPI::Token') and $_[1]->content eq '...' } ); + is( ref $tokens, 'ARRAY', "ellipsis found in sub body" ); + is( @$tokens, 1, "ellipsis found exactly once in sub body" ); + isa_ok( $tokens->[0], 'PPI::Token::Ellipsis', "ellipsis in sub body" ); +} + +ELLIPSIS_SIGNIFICANT: { + my $doc = safe_new \'...;'; + my $tokens = $doc->find( sub { $_[1]->isa('PPI::Token') and $_[1]->content eq '...' } ); + is( ref $tokens, 'ARRAY', "ellipsis found for significance test" ); + ok( $tokens->[0]->significant, "ellipsis is significant" ); +} + +ELLIPSIS_STATEMENT_BREAK: { + my $doc = safe_new \'...;'; + + local $TODO = "ellipsis token type not yet implemented"; + + my $stmts = $doc->find('Statement::Break'); + is( ref $stmts, 'ARRAY', "'...' creates a Statement::Break" ); + is( ref $stmts eq 'ARRAY' ? scalar @$stmts : 0, 1, "exactly one Statement::Break" ); +} + +ELLIPSIS_IN_SUB_STATEMENT: { + my $doc = safe_new \'sub foo { ... }'; + + local $TODO = "ellipsis token type not yet implemented"; + + my $stmts = $doc->find('Statement::Break'); + is( ref $stmts, 'ARRAY', "ellipsis in sub creates Statement::Break" ); + is( ref $stmts eq 'ARRAY' ? scalar @$stmts : 0, 1, "exactly one Statement::Break in sub body" ); +} + +RANGE_OPERATOR_UNCHANGED: { + my $doc = safe_new \'my @a = 1..10;'; + my $ops = $doc->find('Token::Operator'); + is( ref $ops, 'ARRAY', "'..' still found as operator" ); + my @dots = grep { $_->content eq '..' } @$ops; + is( scalar @dots, 1, "range operator '..' found exactly once" ); +} + +CONCAT_OPERATOR_UNCHANGED: { + my $doc = safe_new \'my $x = "a" . "b";'; + my $ops = $doc->find('Token::Operator'); + is( ref $ops, 'ARRAY', "'.' still found as operator" ); + my @dots = grep { $_->content eq '.' } @$ops; + is( scalar @dots, 1, "concat operator '.' found exactly once" ); +} + +ROUND_TRIP: { + my $code = "sub foo { ... }\n"; + my $doc = safe_new \$code; + is( $doc->serialize, $code, "round-trip preserves ellipsis code" ); +} + +ELLIPSIS_WITH_SEMICOLON: { + my $doc = safe_new \'...; print "hello\n";'; + + local $TODO = "ellipsis token type not yet implemented"; + + my $tokens = $doc->find( sub { $_[1]->isa('PPI::Token') and $_[1]->content eq '...' } ); + is( ref $tokens, 'ARRAY', "ellipsis found before semicolon" ); + isa_ok( $tokens->[0], 'PPI::Token::Ellipsis', "ellipsis before semicolon" ); +} From f05bc750a7e2e7fffa6b97609396cd5516a96edc Mon Sep 17 00:00:00 2001 From: Toddr Bot Date: Mon, 27 Apr 2026 08:30:44 +0000 Subject: [PATCH 2/2] fix: parse ellipsis '...' as PPI::Token::Ellipsis, not Operator (#41) The Perl 5.12 ellipsis statement '...' (yada yada) is a statement, not an operator. Create PPI::Token::Ellipsis token type and make the Lexer wrap it in PPI::Statement::Break (like die/return/goto). - Add lib/PPI/Token/Ellipsis.pm inheriting from PPI::Token - Convert '...' from Operator to Ellipsis during tokenizer finalization - Register '...' in Lexer's %STATEMENT_CLASSES as Statement::Break - Update class hierarchy docs in PPI.pm and Token.pm - Update existing tests to expect new token/statement types - Remove $TODO markers from ellipsis tests Co-Authored-By: Claude Opus 4.6 --- lib/PPI.pm | 1 + lib/PPI/Lexer.pm | 1 + lib/PPI/Token.pm | 1 + lib/PPI/Token/Ellipsis.pm | 60 +++++++++++++++++++++++ lib/PPI/Token/Operator.pm | 6 ++- t/data/08_regression/35_attr_perlsub.dump | 4 +- t/ppi_token_ellipsis.t | 16 +----- t/ppi_token_operator.t | 24 ++++++--- t/signature_details.t | 4 +- 9 files changed, 92 insertions(+), 25 deletions(-) create mode 100644 lib/PPI/Token/Ellipsis.pm diff --git a/lib/PPI.pm b/lib/PPI.pm index b072080e..f07f34fa 100644 --- a/lib/PPI.pm +++ b/lib/PPI.pm @@ -423,6 +423,7 @@ based on inheritance. PPI::Token::Magic PPI::Token::ArrayIndex PPI::Token::Operator + PPI::Token::Ellipsis PPI::Token::Quote PPI::Token::Quote::Single PPI::Token::Quote::Double diff --git a/lib/PPI/Lexer.pm b/lib/PPI/Lexer.pm index 5bdeb9de..f4295925 100644 --- a/lib/PPI/Lexer.pm +++ b/lib/PPI/Lexer.pm @@ -370,6 +370,7 @@ my %STATEMENT_CLASSES = ( 'last' => 'PPI::Statement::Break', 'return' => 'PPI::Statement::Break', 'goto' => 'PPI::Statement::Break', + '...' => 'PPI::Statement::Break', # Special sections of the file '__DATA__' => 'PPI::Statement::Data', diff --git a/lib/PPI/Token.pm b/lib/PPI/Token.pm index e64d9e50..0a3686f8 100644 --- a/lib/PPI/Token.pm +++ b/lib/PPI/Token.pm @@ -62,6 +62,7 @@ use PPI::Token::Regexp::Match (); use PPI::Token::Regexp::Substitute (); use PPI::Token::Regexp::Transliterate (); use PPI::Token::Operator (); +use PPI::Token::Ellipsis (); use PPI::Token::Cast (); use PPI::Token::Structure (); use PPI::Token::Label (); diff --git a/lib/PPI/Token/Ellipsis.pm b/lib/PPI/Token/Ellipsis.pm new file mode 100644 index 00000000..16e20a97 --- /dev/null +++ b/lib/PPI/Token/Ellipsis.pm @@ -0,0 +1,60 @@ +package PPI::Token::Ellipsis; + +=pod + +=head1 NAME + +PPI::Token::Ellipsis - The Perl 5.12+ ellipsis statement token + +=head1 INHERITANCE + + PPI::Token::Ellipsis + isa PPI::Token + isa PPI::Element + +=head1 DESCRIPTION + +C represents the C<...> (yada yada) statement +introduced in Perl 5.12. When executed, it throws an +C<"Unimplemented"> exception. + +Although C<...> looks superficially like an operator, it is really +a statement unto itself and does not operate on any values. + +=head1 METHODS + +There are no additional methods beyond those provided by the parent +L and L classes. + +=cut + +use strict; +use PPI::Token (); + +our $VERSION = '1.292'; + +our @ISA = "PPI::Token"; + +1; + +=pod + +=head1 SUPPORT + +See the L in the main module. + +=head1 AUTHOR + +Adam Kennedy Eadamk@cpan.orgE + +=head1 COPYRIGHT + +Copyright 2001 - 2011 Adam Kennedy. + +This program is free software; you can redistribute +it and/or modify it under the same terms as Perl itself. + +The full text of the license can be found in the +LICENSE file included with this module. + +=cut diff --git a/lib/PPI/Token/Operator.pm b/lib/PPI/Token/Operator.pm index e5e22bd3..d8ec882c 100644 --- a/lib/PPI/Token/Operator.pm +++ b/lib/PPI/Token/Operator.pm @@ -18,7 +18,7 @@ PPI::Token::Operator - Token class for operators ++ -- ** ! ~ + - =~ !~ * / % x << >> lt gt le ge cmp ~~ - == != <=> . .. ... , + == != <=> . .. , & | ^ && || // ? : **= += -= .= *= /= %= x= &= |= ^= <<= >>= &&= @@ -95,6 +95,10 @@ sub __TOKENIZER__on_char { $t->{class} = $t->{token}->set_class('QuoteLike::Readline') if $content eq '<>' or $content eq '<<>>'; + # Handle the ellipsis (yada yada) statement + $t->{class} = $t->{token}->set_class('Ellipsis') + if $content eq '...'; + # Finalize normally $t->_finalize_token->__TOKENIZER__on_char( $t ); } diff --git a/t/data/08_regression/35_attr_perlsub.dump b/t/data/08_regression/35_attr_perlsub.dump index 8aaca43c..51fd194c 100644 --- a/t/data/08_regression/35_attr_perlsub.dump +++ b/t/data/08_regression/35_attr_perlsub.dump @@ -26,8 +26,8 @@ PPI::Document PPI::Token::Whitespace ' ' PPI::Structure::Block { ... } PPI::Token::Whitespace ' ' - PPI::Statement - PPI::Token::Operator '...' + PPI::Statement::Break + PPI::Token::Ellipsis '...' PPI::Token::Whitespace ' ' PPI::Token::Whitespace '\n' PPI::Statement::Sub diff --git a/t/ppi_token_ellipsis.t b/t/ppi_token_ellipsis.t index 74796106..ff2a1978 100644 --- a/t/ppi_token_ellipsis.t +++ b/t/ppi_token_ellipsis.t @@ -9,13 +9,9 @@ use Test::More tests => 38 + ($ENV{AUTHOR_TESTING} ? 1 : 0); use PPI (); use Helper 'safe_new'; -our $TODO; - ELLIPSIS_TOKEN_TYPE: { my $doc = safe_new \'...;'; - local $TODO = "ellipsis token type not yet implemented"; - my $tokens = $doc->find( sub { $_[1]->isa('PPI::Token') and $_[1]->content eq '...' } ); is( ref $tokens, 'ARRAY', "'...' token found" ); is( @$tokens, 1, "'...' found exactly once" ); @@ -26,8 +22,6 @@ ELLIPSIS_TOKEN_TYPE: { ELLIPSIS_IN_SUB: { my $doc = safe_new \'sub foo { ... }'; - local $TODO = "ellipsis token type not yet implemented"; - my $tokens = $doc->find( sub { $_[1]->isa('PPI::Token') and $_[1]->content eq '...' } ); is( ref $tokens, 'ARRAY', "ellipsis found in sub body" ); is( @$tokens, 1, "ellipsis found exactly once in sub body" ); @@ -44,21 +38,17 @@ ELLIPSIS_SIGNIFICANT: { ELLIPSIS_STATEMENT_BREAK: { my $doc = safe_new \'...;'; - local $TODO = "ellipsis token type not yet implemented"; - my $stmts = $doc->find('Statement::Break'); is( ref $stmts, 'ARRAY', "'...' creates a Statement::Break" ); - is( ref $stmts eq 'ARRAY' ? scalar @$stmts : 0, 1, "exactly one Statement::Break" ); + is( @$stmts, 1, "exactly one Statement::Break" ); } ELLIPSIS_IN_SUB_STATEMENT: { my $doc = safe_new \'sub foo { ... }'; - local $TODO = "ellipsis token type not yet implemented"; - my $stmts = $doc->find('Statement::Break'); is( ref $stmts, 'ARRAY', "ellipsis in sub creates Statement::Break" ); - is( ref $stmts eq 'ARRAY' ? scalar @$stmts : 0, 1, "exactly one Statement::Break in sub body" ); + is( @$stmts, 1, "exactly one Statement::Break in sub body" ); } RANGE_OPERATOR_UNCHANGED: { @@ -86,8 +76,6 @@ ROUND_TRIP: { ELLIPSIS_WITH_SEMICOLON: { my $doc = safe_new \'...; print "hello\n";'; - local $TODO = "ellipsis token type not yet implemented"; - my $tokens = $doc->find( sub { $_[1]->isa('PPI::Token') and $_[1]->content eq '...' } ); is( ref $tokens, 'ARRAY', "ellipsis found before semicolon" ); isa_ok( $tokens->[0], 'PPI::Token::Ellipsis', "ellipsis before semicolon" ); diff --git a/t/ppi_token_operator.t b/t/ppi_token_operator.t index 45839322..f8a95b4a 100644 --- a/t/ppi_token_operator.t +++ b/t/ppi_token_operator.t @@ -26,10 +26,18 @@ FIND_ONE_OP: { PARSE_ALL_OPERATORS: { foreach my $op ( sort keys %OPERATOR ) { - my $source = $op eq '<>' || $op eq '<<>>' ? $op . ';' : "\$foo $op 2;"; + my $source = $op eq '<>' || $op eq '<<>>' + ? $op . ';' + : $op eq '...' + ? '...;' + : "\$foo $op 2;"; my $doc = safe_new \$source; - my $ops = $doc->find( $op eq '<<>>' || $op eq '<>' - ? 'Token::QuoteLike::Readline' : 'Token::Operator' ); + my $find_class = $op eq '<<>>' || $op eq '<>' + ? 'Token::QuoteLike::Readline' + : $op eq '...' + ? 'Token::Ellipsis' + : 'Token::Operator'; + my $ops = $doc->find( $find_class ); is( ref $ops, 'ARRAY', "operator $op found operators" ); is( @$ops, 1, "operator $op found exactly once" ); is( $ops->[0]->content(), $op, "operator $op operator text matches" ); @@ -519,8 +527,12 @@ OPERATOR_X: { } $code .= $operator; - push @expected, ( ($operator eq '<<>>' || $operator eq '<>' ? - 'PPI::Token::QuoteLike::Readline' : 'PPI::Token::Operator') => $operator ); + my $op_class = $operator eq '<<>>' || $operator eq '<>' + ? 'PPI::Token::QuoteLike::Readline' + : $operator eq '...' + ? 'PPI::Token::Ellipsis' + : 'PPI::Token::Operator'; + push @expected, ( $op_class => $operator ); if ( $operator =~ /\w$/ || $operator eq '<<' ) { # want << operator, not heredoc $code .= ' '; @@ -528,7 +540,7 @@ OPERATOR_X: { } $code .= 'x3'; my $desc; - if ( $operator eq '--' || $operator eq '++' || $operator eq '<>' || $operator eq '<<>>' ) { + if ( $operator eq '--' || $operator eq '++' || $operator eq '<>' || $operator eq '<<>>' || $operator eq '...' ) { push @expected, ( 'PPI::Token::Operator' => 'x' ); push @expected, ( 'PPI::Token::Number' => '3' ); $desc = "operator $operator does not imply following 'x' is a word"; diff --git a/t/signature_details.t b/t/signature_details.t index c2fc9802..0744790d 100644 --- a/t/signature_details.t +++ b/t/signature_details.t @@ -411,8 +411,8 @@ END_PERL 'PPI::Token::Structure', ')', 'PPI::Structure::Block', '{ ... }', 'PPI::Token::Structure', '{', - 'PPI::Statement', '...', - 'PPI::Token::Operator', '...', + 'PPI::Statement::Break', '...', + 'PPI::Token::Ellipsis', '...', 'PPI::Token::Structure', '}', ], "complex signature example";