fix: parse ellipsis '...' as PPI::Token::Ellipsis (#41)

lib/PPI.pm

@@ -423,6 +423,7 @@ based on inheritance.
             PPI::Token::Magic
          PPI::Token::ArrayIndex
          PPI::Token::Operator
+         PPI::Token::Ellipsis
          PPI::Token::Quote
             PPI::Token::Quote::Single
             PPI::Token::Quote::Double

lib/PPI/Lexer.pm

@@ -370,6 +370,7 @@ my %STATEMENT_CLASSES = (
 	'last'      => 'PPI::Statement::Break',
 	'return'    => 'PPI::Statement::Break',
 	'goto'      => 'PPI::Statement::Break',
+	'...'       => 'PPI::Statement::Break',
 
 	# Special sections of the file
 	'__DATA__'  => 'PPI::Statement::Data',

lib/PPI/Token.pm

@@ -62,6 +62,7 @@ use PPI::Token::Regexp::Match         ();
 use PPI::Token::Regexp::Substitute    ();
 use PPI::Token::Regexp::Transliterate ();
 use PPI::Token::Operator              ();
+use PPI::Token::Ellipsis              ();
 use PPI::Token::Cast                  ();
 use PPI::Token::Structure             ();
 use PPI::Token::Label                 ();

lib/PPI/Token/Ellipsis.pm

@@ -0,0 +1,60 @@
+package PPI::Token::Ellipsis;
+
+=pod
+
+=head1 NAME
+
+PPI::Token::Ellipsis - The Perl 5.12+ ellipsis statement token
+
+=head1 INHERITANCE
+
+  PPI::Token::Ellipsis
+  isa PPI::Token
+      isa PPI::Element
+
+=head1 DESCRIPTION
+
+C<PPI::Token::Ellipsis> represents the C<...> (yada yada) statement
+introduced in Perl 5.12. When executed, it throws an
+C<"Unimplemented"> exception.
+
+Although C<...> looks superficially like an operator, it is really
+a statement unto itself and does not operate on any values.
+
+=head1 METHODS
+
+There are no additional methods beyond those provided by the parent
+L<PPI::Token> and L<PPI::Element> classes.
+
+=cut
+
+use strict;
+use PPI::Token ();
+
+our $VERSION = '1.292';
+
+our @ISA = "PPI::Token";
+
+1;
+
+=pod
+
+=head1 SUPPORT
+
+See the L<support section|PPI/SUPPORT> in the main module.
+
+=head1 AUTHOR
+
+Adam Kennedy E<lt>adamk@cpan.orgE<gt>
+
+=head1 COPYRIGHT
+
+Copyright 2001 - 2011 Adam Kennedy.
+
+This program is free software; you can redistribute
+it and/or modify it under the same terms as Perl itself.
+
+The full text of the license can be found in the
+LICENSE file included with this module.
+
+=cut

lib/PPI/Token/Operator.pm

@@ -18,7 +18,7 @@ PPI::Token::Operator - Token class for operators
   ++   --   **   !    ~    +    -
   =~   !~   *    /    %    x
   <<   >>   lt   gt   le   ge   cmp  ~~
-  ==   !=   <=>  .    ..   ...  ,
+  ==   !=   <=>  .    ..   ,
   &    |    ^    &&   ||   //
   ?    :    **=  +=   -=   .=   *=   /=
   %=   x=   &=   |=   ^=   <<=  >>=  &&=
@@ -95,6 +95,10 @@ sub __TOKENIZER__on_char {
 	$t->{class} = $t->{token}->set_class('QuoteLike::Readline')
 		if $content eq '<>' or $content eq '<<>>';
 
+	# Handle the ellipsis (yada yada) statement
+	$t->{class} = $t->{token}->set_class('Ellipsis')
+		if $content eq '...';
+
 	# Finalize normally
 	$t->_finalize_token->__TOKENIZER__on_char( $t );
 }

t/data/08_regression/35_attr_perlsub.dump

@@ -26,8 +26,8 @@ PPI::Document
     PPI::Token::Whitespace  	' '
     PPI::Structure::Block  	{ ... }
       PPI::Token::Whitespace  	' '
-      PPI::Statement
-        PPI::Token::Operator  	'...'
+      PPI::Statement::Break
+        PPI::Token::Ellipsis  	'...'
       PPI::Token::Whitespace  	' '
   PPI::Token::Whitespace  	'\n'
   PPI::Statement::Sub

t/ppi_token_ellipsis.t

@@ -0,0 +1,82 @@
+#!/usr/bin/perl
+
+# Unit testing for PPI::Token::Ellipsis
+
+use lib 't/lib';
+use PPI::Test::pragmas;
+use Test::More tests => 38 + ($ENV{AUTHOR_TESTING} ? 1 : 0);
+
+use PPI ();
+use Helper 'safe_new';
+
+ELLIPSIS_TOKEN_TYPE: {
+	my $doc = safe_new \'...;';
+
+	my $tokens = $doc->find( sub { $_[1]->isa('PPI::Token') and $_[1]->content eq '...' } );
+	is( ref $tokens, 'ARRAY', "'...' token found" );
+	is( @$tokens, 1, "'...' found exactly once" );
+	isa_ok( $tokens->[0], 'PPI::Token::Ellipsis', "'...' token" );
+	ok( !$tokens->[0]->isa('PPI::Token::Operator'), "'...' is not a Token::Operator" );
+}
+
+ELLIPSIS_IN_SUB: {
+	my $doc = safe_new \'sub foo { ... }';
+
+	my $tokens = $doc->find( sub { $_[1]->isa('PPI::Token') and $_[1]->content eq '...' } );
+	is( ref $tokens, 'ARRAY', "ellipsis found in sub body" );
+	is( @$tokens, 1, "ellipsis found exactly once in sub body" );
+	isa_ok( $tokens->[0], 'PPI::Token::Ellipsis', "ellipsis in sub body" );
+}
+
+ELLIPSIS_SIGNIFICANT: {
+	my $doc = safe_new \'...;';
+	my $tokens = $doc->find( sub { $_[1]->isa('PPI::Token') and $_[1]->content eq '...' } );
+	is( ref $tokens, 'ARRAY', "ellipsis found for significance test" );
+	ok( $tokens->[0]->significant, "ellipsis is significant" );
+}
+
+ELLIPSIS_STATEMENT_BREAK: {
+	my $doc = safe_new \'...;';
+
+	my $stmts = $doc->find('Statement::Break');
+	is( ref $stmts, 'ARRAY', "'...' creates a Statement::Break" );
+	is( @$stmts, 1, "exactly one Statement::Break" );
+}
+
+ELLIPSIS_IN_SUB_STATEMENT: {
+	my $doc = safe_new \'sub foo { ... }';
+
+	my $stmts = $doc->find('Statement::Break');
+	is( ref $stmts, 'ARRAY', "ellipsis in sub creates Statement::Break" );
+	is( @$stmts, 1, "exactly one Statement::Break in sub body" );
+}
+
+RANGE_OPERATOR_UNCHANGED: {
+	my $doc = safe_new \'my @a = 1..10;';
+	my $ops = $doc->find('Token::Operator');
+	is( ref $ops, 'ARRAY', "'..' still found as operator" );
+	my @dots = grep { $_->content eq '..' } @$ops;
+	is( scalar @dots, 1, "range operator '..' found exactly once" );
+}
+
+CONCAT_OPERATOR_UNCHANGED: {
+	my $doc = safe_new \'my $x = "a" . "b";';
+	my $ops = $doc->find('Token::Operator');
+	is( ref $ops, 'ARRAY', "'.' still found as operator" );
+	my @dots = grep { $_->content eq '.' } @$ops;
+	is( scalar @dots, 1, "concat operator '.' found exactly once" );
+}
+
+ROUND_TRIP: {
+	my $code = "sub foo { ... }\n";
+	my $doc = safe_new \$code;
+	is( $doc->serialize, $code, "round-trip preserves ellipsis code" );
+}
+
+ELLIPSIS_WITH_SEMICOLON: {
+	my $doc = safe_new \'...; print "hello\n";';
+
+	my $tokens = $doc->find( sub { $_[1]->isa('PPI::Token') and $_[1]->content eq '...' } );
+	is( ref $tokens, 'ARRAY', "ellipsis found before semicolon" );
+	isa_ok( $tokens->[0], 'PPI::Token::Ellipsis', "ellipsis before semicolon" );
+}

t/ppi_token_operator.t

@@ -26,10 +26,18 @@ FIND_ONE_OP: {
 
 PARSE_ALL_OPERATORS: {
 	foreach my $op ( sort keys %OPERATOR ) {
-		my $source = $op eq '<>' || $op eq '<<>>' ? $op . ';' : "\$foo $op 2;";
+		my $source = $op eq '<>' || $op eq '<<>>'
+			? $op . ';'
+			: $op eq '...'
+			? '...;'
+			: "\$foo $op 2;";
 		my $doc = safe_new \$source;
-		my $ops = $doc->find( $op eq '<<>>' || $op eq '<>'
-			? 'Token::QuoteLike::Readline' : 'Token::Operator' );
+		my $find_class = $op eq '<<>>' || $op eq '<>'
+			? 'Token::QuoteLike::Readline'
+			: $op eq '...'
+			? 'Token::Ellipsis'
+			: 'Token::Operator';
+		my $ops = $doc->find( $find_class );
 		is( ref $ops, 'ARRAY', "operator $op found operators" );
 		is( @$ops, 1, "operator $op found exactly once" );
 		is( $ops->[0]->content(), $op, "operator $op operator text matches" );
@@ -519,16 +527,20 @@ OPERATOR_X: {
 		}
 
 		$code .= $operator;
-		push @expected, ( ($operator eq '<<>>' || $operator eq '<>' ?
-			'PPI::Token::QuoteLike::Readline' : 'PPI::Token::Operator') => $operator );
+		my $op_class = $operator eq '<<>>' || $operator eq '<>'
+			? 'PPI::Token::QuoteLike::Readline'
+			: $operator eq '...'
+			? 'PPI::Token::Ellipsis'
+			: 'PPI::Token::Operator';
+		push @expected, ( $op_class => $operator );
 
 		if ( $operator =~ /\w$/ || $operator eq '<<' ) {  # want << operator, not heredoc
 			$code .= ' ';
 			push @expected, ( 'PPI::Token::Whitespace' => ' ' );
 		}
 		$code .= 'x3';
 		my $desc;
-		if ( $operator eq '--' || $operator eq '++' || $operator eq '<>' || $operator eq '<<>>' ) {
+		if ( $operator eq '--' || $operator eq '++' || $operator eq '<>' || $operator eq '<<>>' || $operator eq '...' ) {
 			push @expected, ( 'PPI::Token::Operator' => 'x' );
 			push @expected, ( 'PPI::Token::Number' => '3' );
 			$desc = "operator $operator does not imply following 'x' is a word";

t/signature_details.t

@@ -411,8 +411,8 @@ END_PERL
 		'PPI::Token::Structure',      ')',
 		'PPI::Structure::Block',      '{ ... }',
 		'PPI::Token::Structure',      '{',
-		'PPI::Statement',             '...',
-		'PPI::Token::Operator',       '...',
+		'PPI::Statement::Break',      '...',
+		'PPI::Token::Ellipsis',       '...',
 		'PPI::Token::Structure',      '}',
 	  ],
 	  "complex signature example";