diff --git a/.coveragerc b/.coveragerc index 3c3efd737..de8cf5fb1 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,5 +1,3 @@ [report] -# Verifier is used for testing only. omit = */__main__.py - */verifier.py diff --git a/.flake8 b/.flake8 index 06d70e9bd..315910712 100644 --- a/.flake8 +++ b/.flake8 @@ -1,8 +1,17 @@ [flake8] ignore = - # indentation is not a multiple of four, - E111,E114, + # 'toml' imported but unused + F401, + # closing bracket does not match visual indentation + E124, + # continuation line over-indented for hanging indent + E126, # visually indented line with same indent as next logical line, - E129 + E129, + # line break before binary operator + W503, + # line break after binary operator + W504 -max-line-length=80 +indent-size = 2 +max-line-length = 80 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..5bb982c51 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +.python-version eol=lf diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..c963e3c8b --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,12 @@ +version: 2 +updates: + + - package-ecosystem: "github-actions" + commit-message: + include: "scope" + prefix: "Actions" + directory: "/" + labels: + - "enhancement" + schedule: + interval: "weekly" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..754ff1555 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,37 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Test with pytest + +on: + pull_request: + push: + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: ["3.8", "3.11", "3.12"] # no particular need for 3.9 or 3.10 + os: [macos-latest, ubuntu-latest, windows-latest] + steps: + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v4.6.0 + with: + python-version: ${{ matrix.python-version }} + - name: Upgrade pip + run: >- + python -m pip install + --upgrade + --disable-pip-version-check + pip + - name: Perform package installs + run: >- + pip install + . + pytest + pytest-cov + - name: Test with pytest + run: pytest diff --git a/.github/workflows/pre-commit-autoupdate.yml b/.github/workflows/pre-commit-autoupdate.yml new file mode 100644 index 000000000..91b8e4614 --- /dev/null +++ b/.github/workflows/pre-commit-autoupdate.yml @@ -0,0 +1,63 @@ +# Copyright (c) 2023 Sebastian Pipping +# Licensed under the Apache License Version 2.0 + +name: Keep pre-commit hooks up to date + +on: + schedule: + - cron: '0 16 * * 5' # Every Friday 4pm + workflow_dispatch: + +# NOTE: This will drop all permissions from GITHUB_TOKEN except metadata read, +# and then (re)add the ones listed below: +permissions: + contents: write + pull-requests: write + +jobs: + pre_commit_autoupdate: + name: Detect outdated pre-commit hooks + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 + + - name: Set up Python 3.11 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 + with: + python-version: 3.11 + + - name: Install pre-commit + run: |- + pip install \ + --disable-pip-version-check \ + --no-warn-script-location \ + --user \ + pre-commit + echo "PATH=${HOME}/.local/bin:${PATH}" >> "${GITHUB_ENV}" + + - name: Check for outdated hooks + run: |- + pre-commit autoupdate + git diff -- .pre-commit-config.yaml + + - name: Create pull request from changes (if any) + id: create-pull-request + uses: peter-evans/create-pull-request@5e914681df9dc83aa4e4905692ca88beb2f9e91f # v7.0.5 + with: + author: 'pre-commit ' + base: main + body: |- + For your consideration. + + :warning: Please **CLOSE AND RE-OPEN** this pull request so that [further workflow runs get triggered](https://github.com/peter-evans/create-pull-request/blob/main/docs/concepts-guidelines.md#triggering-further-workflow-runs) for this pull request. + branch: precommit-autoupdate + commit-message: "pre-commit: Autoupdate" + delete-branch: true + draft: true + labels: enhancement + title: "pre-commit: Autoupdate" + + - name: Log pull request URL + if: "${{ steps.create-pull-request.outputs.pull-request-url }}" + run: | + echo "Pull request URL is: ${{ steps.create-pull-request.outputs.pull-request-url }}" diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 000000000..c01223a6d --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,37 @@ +# Copyright (c) 2023 Sebastian Pipping +# Licensed under the Apache License Version 2.0 + +name: Run pre-commit + +# Drop permissions to minimum for security +permissions: + contents: read + +on: + pull_request: + push: + schedule: + - cron: '0 2 * * 5' # Every Friday at 2am + workflow_dispatch: + +jobs: + pre_commit_run: + name: Run pre-commit + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 + + - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 + with: + python-version: 3.11 + + - name: Install yapf (to be available to pre-commit) + run: |- + pip install \ + --disable-pip-version-check \ + --no-warn-script-location \ + --user \ + . + echo "PATH=${HOME}/.local/bin:${PATH}" >> "${GITHUB_ENV}" + + - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 diff --git a/.gitignore b/.gitignore index 3fd6b2b3b..6a6928eae 100644 --- a/.gitignore +++ b/.gitignore @@ -13,8 +13,9 @@ *~ # Merge files created by git. *.orig -# Byte compiled python modules. +# Compiled python. *.pyc +*.pickle # vim swap files .*.sw? .sw? @@ -32,3 +33,13 @@ /dist /.tox /yapf.egg-info + +# IDEs +/.idea +/.vscode/settings.json + +# Virtual Environment +/.venv*/ + +# Worktrees +/.wt diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 000000000..d9ce8b88c --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,6 @@ +[settings] +force_single_line=true +known_third_party=yapf_third_party +known_yapftests=yapftests + +sections=FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER,YAPFTESTS diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..75be82045 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,36 @@ +# File introduces automated checks triggered on git events +# to enable run `pip install pre-commit && pre-commit install` + +repos: + - repo: https://github.com/pycqa/isort + rev: 6.0.0 + hooks: + - id: isort + name: isort (python) + - repo: local + hooks: + - id: yapf + name: yapf + language: python + entry: yapf + args: [-i] + types: [python] + - repo: https://github.com/pycqa/flake8 + rev: 7.1.1 + hooks: + - id: flake8 + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: trailing-whitespace + - id: check-docstring-first + - id: check-added-large-files + - id: check-yaml + - id: debug-statements + - id: check-merge-conflict + - id: double-quote-string-fixer + - id: end-of-file-fixer + - repo: meta + hooks: + - id: check-hooks-apply + - id: check-useless-excludes diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml new file mode 100644 index 000000000..e834fc4db --- /dev/null +++ b/.pre-commit-hooks.yaml @@ -0,0 +1,21 @@ +# File configures YAPF to be used as a git hook with https://github.com/pre-commit/pre-commit + +- id: yapf + name: yapf + description: "A formatter for Python files." + entry: yapf + args: [-i] #inplace + language: python + types: [python] + +- id: yapf-diff + name: yapf-diff + description: "A formatter for Python files. (formats only changes included in commit)" + always_run: true + language: python + pass_filenames: false + stages: [pre-commit] + entry: | + bash -c "git diff -U0 --no-color --relative HEAD \ + | yapf-diff \ + | tee >(git apply --allow-empty -p0)" diff --git a/.python-version b/.python-version new file mode 100644 index 000000000..f4bd916f2 --- /dev/null +++ b/.python-version @@ -0,0 +1,5 @@ +3.7.9 +3.8.10 +3.9.13 +3.10.11 +3.11.5 diff --git a/.style.yapf b/.style.yapf index 823a973ea..fdd07237c 100644 --- a/.style.yapf +++ b/.style.yapf @@ -1,3 +1,2 @@ [style] -# YAPF uses the chromium style -based_on_style = chromium +based_on_style = yapf diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 87965b4ab..000000000 --- a/.travis.yml +++ /dev/null @@ -1,27 +0,0 @@ -language: python - -python: - - 2.7 - - 3.4 - - 3.5 - - nightly - -matrix: - allow_failures: - - python: nightly - include: - - python: 2.7 - env: SCA=true - - python: 3.5 - env: SCA=true - -install: - - if [ -z "$SCA" ]; then pip install --quiet coveralls; else echo skip; fi - - if [ -n "$SCA" ]; then python setup.py develop; else echo skip; fi - -script: - - if [ -n "$SCA" ]; then yapf --diff --recursive . || exit; else echo skip; fi - - if [ -z "$SCA" ]; then nosetests --with-coverage --cover-package=yapf; else echo skip; fi - -after_success: - - coveralls diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 000000000..f3f2fe836 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,15 @@ +{ + "recommendations": [ + "eeyore.yapf", + "dangmai.workspace-default-settings", + "ms-python.flake8", + "ms-python.isort", + "ms-python.python", + ], + // These are remarked as extenstions you should disable for this workspace. + // VSCode does not support disabling extensions via workspace config files. + "unwantedRecommendations": [ + "ms-python.black-formatter", + "ms-python.pylint" + ] +} diff --git a/.vscode/settings.default.json b/.vscode/settings.default.json new file mode 100644 index 000000000..502f59d95 --- /dev/null +++ b/.vscode/settings.default.json @@ -0,0 +1,33 @@ +{ + "editor.codeActionsOnSave": { + "source.organizeImports": true + }, + "files.insertFinalNewline": true, + "files.trimFinalNewlines": true, + "[python]": { + "diffEditor.ignoreTrimWhitespace": false, + "editor.defaultFormatter": "eeyore.yapf", + "editor.formatOnSaveMode": "file", + "editor.formatOnSave": true, + "editor.wordBasedSuggestions": false, + "files.trimTrailingWhitespace": true, + }, + "python.analysis.extraPaths": [ + "./third_party" + ], + "python.analysis.typeCheckingMode": "basic", + "python.languageServer": "Pylance", + "files.exclude": { + "**/*$py.class": true + }, + "json.schemas": [ + { + "fileMatch": [ + "/.vscode/settings.default.json" + ], + "url": "vscode://schemas/settings/folder" + } + ], + "workspace-default-settings.runOnActivation": true, + "workspace-default-settings.jsonIndentation": 4 +} diff --git a/CHANGELOG b/CHANGELOG deleted file mode 100644 index d331f8d90..000000000 --- a/CHANGELOG +++ /dev/null @@ -1,225 +0,0 @@ -# Change Log -# All notable changes to this project will be documented in this file. -# This project adheres to [Semantic Versioning](http://semver.org/). - -## [0.11.1] UNRELEASED -### Fixed -- Enforce splitting each element in a dictionary if comma terminated. -- It's okay to split in the middle of a dotted name if the whole expression is - going to go over the column limit. - -## [0.11.0] 2016-07-17 -### Added -- The COALESCE_BRACKETS knob prevents splitting consecutive brackets when - DEDENT_CLOSING_BRACKETS is set. -- Don't count "pylint" directives as exceeding the column limit. - -### Changed -- We split all of the arguments to a function call if there's a named argument. - In this case, we want to split after the opening bracket too. This makes - things look a bit better. - -### Fixed -- When retaining format of a multiline string with Chromium style, make sure - that the multiline string doesn't mess up where the following comma ends up. -- Correct for when 'lib2to3' smooshes comments together into the same DEDENT - node. - -## [0.10.0] 2016-06-14 -### Added -- Add a knob, 'USE_TABS', to allow using tabs for indentation. - -### Changed -- Performance enhancements. - -### Fixed -- Don't split an import list if it's not surrounded by parentheses. - -## [0.9.0] 2016-05-29 -### Added -- Added a knob (SPLIT_PENALTY_BEFORE_IF_EXPR) to adjust the split penalty - before an if expression. This allows the user to place a list comprehension - all on one line. -- Added a knob (SPLIT_BEFORE_FIRST_ARGUMENT) that encourages splitting before - the first element of a list of arguments or parameters if they are going to - be split anyway. -- Added a knob (SPLIT_ARGUMENTS_WHEN_COMMA_TERMINATED) splits arguments to a - function if the list is terminated by a comma. - -### Fixed -- Don't split before a first element list argument as we would before a first - element function call. -- Don't penalize when we must split a line. -- Allow splitting before the single argument in a function call. - -## [0.8.2] 2016-05-21 -### Fixed -- Prefer not to split after the opening of a subscript. -- Don't add space before the 'await' keyword if it's preceded by an opening - paren. -- When we're setting the split penalty for a continuous list, we don't want to - mistake a comment at the end of that list as part of the list. -- When calculating blank lines, don't assume the last seen object was a class - or function when we're in a class or function. -- Don't count the closing scope when determining if the current scope is the - last scope on the line. - -## [0.8.1] 2016-05-18 -### Fixed -- 'SPLIT_BEFORE_LOGICAL_OPERATOR' wasn't working correctly. The penalty was - being set incorrectly when it was part of a larger construct. -- Don't separate a keyword, like "await", from a left paren. -- Don't rely upon the original tokens' line number to determine if we should - perform splitting in Facebook mode. The line number isn't the line number of - the reformatted token, but the line number where it was in the original code. - Instead, we need to carefully determine if the line is liabel to be split and - act accordingly. - -## [0.8.0] 2016-05-10 -### Added -- Add a knob, 'SPACES_AROUND_POWER_OPERATOR', to allow adding spaces around the - power operator. - -### Fixed -- There shouldn't be a space between a decorator and an intervening comment. -- If we split before a bitwise operator, then we assume that the programmer - knows what they're doing, more or less, and so we enforce a split before said - operator if one exists in the original program. -- Strengthen the bond between a keyword and value argument. -- Don't add a blank line after a multiline string. -- If the "for" part of a list comprehension can exist on the starting line - without going over the column limit, then let it remain there. - -## [0.7.1] 2016-04-21 -### Fixed -- Don't rewrite the file if there are no changes. -- Ensure the proper number of blank lines before an async function. -- Split after a bitwise operator when in PEP 8 mode. -- Retain the splitting within a dictionary data literal between the key and - value. -- Try to keep short function calls all on one line even if they're part of a - larger series of tokens. This stops us from splitting too much. - -## [0.7.0] 2016-04-09 -### Added -- Support for Python 3.5. -- Add 'ALLOW_MULTILINE_LAMBDAS' which allows lambdas to be formatted onto - multiple lines. - -### Fixed -- Lessen penalty for splitting before a dictionary keyword. -- Formatting of trailing comments on disabled formatting lines. -- Disable / enable formatting at end of multi-line comment. - -## [0.6.3] 2016-03-06 -### Changed -- Documentation updated. - -### Fixed -- Fix spacing of multiline comments when formatting is disabled. - -## [0.6.2] 2015-11-01 -### Changed -- Look at the 'setup.cfg' file to see if it contains style information for - YAPF. -- Look at the '~/.config/yapf/style' file to see if it contains global style - information for YAPF. - -### Fixed -- Make lists that can fit on one line more likely to stay together. -- Correct formatting of '*args' and '**kwargs' when there are default values in - the argument list. - -## [0.6.1] 2015-10-24 -### Fixed -- Make sure to align comments in data literals correctly. Also make sure we - don't count a "#." in a string as an i18n comment. -- Retain proper vertical spacing before comments in a data literal. -- Make sure that continuations from a compound statement are distinguished from - the succeeding line. -- Ignore preceding comments when calculating what is a "dictonary maker". -- Add a small penalty for splitting before a closing bracket. -- Ensure that a space is enforced after we remove a pseudo-paren that's between - two names, keywords, numbers, etc. -- Increase the penalty for splitting after a pseudo-paren. This could lead to - less readable code in some circumstances. - -## [0.6.0] 2015-10-18 -### Added -- Add knob to indent the dictionary value if there is a split before it. - -### Changed -- No longer check that a file is a "Python" file unless the '--recursive' flag - is specified. -- No longer allow the user to specify a directory unless the '--recursive' flag - is specified. - -### Fixed -- When determining if we should split a dictionary's value to a new line, use - the longest entry instead of the total dictionary's length. This allows the - formatter to reformat the dictionary in a more consistent manner. -- Improve how list comprehensions are formatted. Make splitting dependent upon - whether the "comp_for" or "comp_if" goes over the column limit. -- Don't over indent if expression hanging indents if we expect to dedent the - closing bracket. -- Improve splitting heuristic when the first argument to a function call is - itself a function call with arguments. In cases like this, the remaining - arguments to the function call would look badly aligned, even though they are - techincally correct (the best kind of correct!). -- Improve splitting heuristic more so that if the first argument to a function - call is a data literal that will go over the column limit, then we want to - split before it. -- Remove spaces around '**' operator. -- Retain formatting of comments in the middle of an expression. -- Don't add a newline to an empty file. -- Over indent a function's parameter list if it's not distinguished from the - body of the function. - -## [0.5.0] 2015-10-11 -### Added -- Add option to exclude files/directories from formatting. -- Add a knob to control whether import names are split after the first '('. - -### Fixed -- Indent the continuation of an if-then statement when it's not distinguished - from the body of the if-then. -- Allow for sensible splitting of array indices where appropriate. -- Prefer to not split before the ending bracket of an atom. This produces - better code in most cases. -- Corrected how horizontal spaces were presevered in a disabled region. - -## [0.4.0] 2015-10-07 -### Added -- Support for dedenting closing brackets, "facebook" style. - -### Fixed -- Formatting of tokens after a multiline string didn't retain their horizontal - spacing. - -## [0.3.1] 2015-09-30 -### Fixed -- Format closing scope bracket correctly when indentation size changes. - -## [0.3.0] 2015-09-20 -### Added -- Return a 2 if the source changed, 1 on error, and 0 for no change. - -### Fixed -- Make sure we format if the "lines" specified are in the middle of a - statement. - -## [0.2.9] - 2015-09-13 -### Fixed -- Formatting of multiple files. It was halting after formatting the first file. - -## [0.2.8] - 2015-09-12 -### Added -- Return a non-zero exit code if the source was changed. -- Add bitwise operator splitting penalty and prefer to split before bitwise - operators. - -### Fixed -- Retain vertical spacing between disabled and enabled lines. -- Split only at start of named assign. -- Retain comment position when formatting is disabled. -- Honor splitting before or after logical ops. diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 000000000..17cc37277 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,927 @@ +# Change Log +# All notable changes to this project will be documented in this file. +# This project adheres to [Semantic Versioning](http://semver.org/). + +## (0.41.0) UNRELEASED +### Added +- New `ALIGN_ASSIGNMENT` and `ALIGN_ASSIGNMENT_RESTART_AFTER_COMMENTS` flags. + `ALIGN_ASSIGNMENT` is a new knob that enables alignment of assignment + operators in consecutive lines. + + A blank line or multiline object (function, multiline dictionary, ...) will + always interrupt the alignment block and start a new one. Comments can also + optionally interrupt the block, a behaviour that can be turned on with + `ALIGN_ASSIGNMENT_RESTART_AFTER_COMMENTS`. + + For example, with `ALIGN_ASSIGNMENT` off: + ``` + val_first = 1 + val_second += 2 + # comment + val_third = 3 + ``` + + will be formatted as: + ``` + val_first = 1 + val_second += 2 + # comment + val_third = 3 + ``` + whereas with `ALIGN_ASSIGNMENT` on and + `ALIGN_ASSIGNMENT_RESTART_AFTER_COMMENTS` off we get: + ``` + val_first = 1 + val_second += 2 + # comment + val_third = 3 + ``` + while if they are both on: + ``` + val_first = 1 + val_second += 2 + # comment + val_third = 3 + ``` + +- New `DISABLE_SPLIT_LIST_WITH_COMMENT` flag. + `DISABLE_SPLIT_LIST_WITH_COMMENT` is a new knob that changes the + behavior of splitting a list when a comment is present inside the list. + + Before, we split a list containing a comment just like we split a list + containing a trailing comma: Each element goes on its own line (unless + `DISABLE_ENDING_COMMA_HEURISTIC` is true). + + This new flag allows you to control the behavior of a list with a comment + *separately* from the behavior when the list contains a trailing comma. + + This mirrors the behavior of clang-format, and is useful for e.g. forming + "logical groups" of elements in a list. + + Without this flag: + + ``` + [ + a, + b, # + c + ] + ``` + + With this flag: + + ``` + [ + a, b, # + c + ] + ``` + + Before we had one flag that controlled two behaviors. + + - `DISABLE_ENDING_COMMA_HEURISTIC=false` (default): + - Split a list that has a trailing comma. + - Split a list that contains a comment. + - `DISABLE_ENDING_COMMA_HEURISTIC=true`: + - Don't split on trailing comma. + - Don't split on comment. + + Now we have two flags. + + - `DISABLE_ENDING_COMMA_HEURISTIC=false` and `DISABLE_SPLIT_LIST_WITH_COMMENT=false` (default): + - Split a list that has a trailing comma. + - Split a list that contains a comment. + Behavior is unchanged from the default before. + - `DISABLE_ENDING_COMMA_HEURISTIC=true` and `DISABLE_SPLIT_LIST_WITH_COMMENT=false` : + - Don't split on trailing comma. + - Do split on comment. **This is a change in behavior from before.** + - `DISABLE_ENDING_COMMA_HEURISTIC=false` and `DISABLE_SPLIT_LIST_WITH_COMMENT=true` : + - Split on trailing comma. + - Don't split on comment. + - `DISABLE_ENDING_COMMA_HEURISTIC=true` and `DISABLE_SPLIT_LIST_WITH_COMMENT=true` : + - Don't split on trailing comma. + - Don't split on comment. + **You used to get this behavior just by setting one flag, but now you have to set both.** + + Note the behavioral change above; if you set + `DISABLE_ENDING_COMMA_HEURISTIC=true` and want to keep the old behavior, you + now also need to set `DISABLE_SPLIT_LIST_WITH_COMMENT=true`. +### Changes +- Remove dependency on importlib-metadata +- Remove dependency on tomli when using >= py311 +- Format '.pyi' type sub files. +### Fixed +- Fix SPLIT_ARGUMENTS_WHEN_COMMA_TERMINATED for one-item named argument lists + by taking precedence over SPLIT_BEFORE_NAMED_ASSIGNS. +- Fix SPLIT_ALL_COMMA_SEPARATED_VALUES and SPLIT_ALL_TOP_LEVEL_COMMA_SEPARATED_VALUES + being too agressive for lambdas and unpacking. + +## [0.40.2] 2023-09-22 +### Changes +- The verification module has been removed. NOTE: this changes the public APIs + by removing the "verify" parameter. +- Changed FORCE_MULTILINE_DICT to override SPLIT_ALL_TOP_LEVEL_COMMA_SEPARATED_VALUES. +- Adopt pyproject.toml (PEP 517) for build system +### Fixed +- Do not treat variables named `match` as the match keyword. +- Fix SPLIT_ARGUMENTS_WHEN_COMMA_TERMINATED for one-item argument lists. +- Fix trailing backslash-newline on Windows when using stdin. + +## [0.40.1] 2023-06-20 +### Fixed +- Corrected bad distribution v0.40.0 package. + +## [0.40.0] 2023-06-13 [YANKED - [#1107](https://github.com/google/yapf/issues/1107)] +### Added +- Support for Python 3.11 +- Add the `--print-modified` flag to print out file names of modified files when + running in in-place mode. +### Changes +- Replace the outdated and no-longer-supported lib2to3 with a fork of blib2to3, + Black's version of lib2to3. +### Removed +- Support for Python versions < 3.7 are no longer supported. + +## [0.33.0] 2023-04-18 [YANKED - [#1154](https://github.com/google/yapf/issues/1154)] +### Added +- Add a new Python parser to generate logical lines. +- Added support for `# fmt: on` and `# fmt: off` pragmas. +### Changes +- Moved 'pytree' parsing tools into its own subdirectory. +- Add support for Python 3.10. +- Format generated dicts with respect to same rules as regular dicts +- Generalized the ending comma heuristic to subscripts. +- Supports "pyproject.toml" by default. +### Fixed +- Split line before all comparison operators. + +## [0.32.0] 2021-12-26 +### Added +- Look at the 'pyproject.toml' file to see if it contains ignore file information + for YAPF. +- New entry point `yapf_api.FormatTree` for formatting lib2to3 concrete + syntax trees. +- Add CI via GitHub Actions. +### Changes +- Change tests to support "pytest". +- Reformat so that "flake8" is happy. +- Use GitHub Actions instead of Travis for CI. +- Clean up the FormatToken interface to limit how much it relies upon the + pytree node object. +- Rename "unwrapped_line" module to "logical_line." +- Rename "UnwrappedLine" class to "LogicalLine." +### Fixed +- Added pyproject extra to install toml package as an optional dependency. +- Enable `BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF` knob for "pep8" style, so + method definitions inside a class are surrounded by a single blank line as + prescribed by PEP8. +- Fixed the '...' token to be spaced after a colon. + +## [0.31.0] 2021-03-14 +### Added +- Renamed 'master' branch to 'main'. +- Add 'BLANK_LINES_BETWEEN_TOP_LEVEL_IMPORTS_AND_VARIABLES' to support setting + a custom number of blank lines between top-level imports and variable + definitions. +- Ignore end of line `# copybara:` directives when checking line length. +- Look at the 'pyproject.toml' file to see if it contains style information for + YAPF. +### Changed +- Do not scan excluded directories. Prior versions would scan an excluded + folder then exclude its contents on a file by file basis. Preventing the + folder being scanned is faster. +### Fixed +- Exclude directories on Windows. + +## [0.30.0] 2020-04-23 +### Added +- Added `SPACES_AROUND_LIST_DELIMITERS`, `SPACES_AROUND_DICT_DELIMITERS`, + and `SPACES_AROUND_TUPLE_DELIMITERS` to add spaces after the opening- + and before the closing-delimiters for lists, dicts, and tuples. +- Adds `FORCE_MULTILINE_DICT` knob to ensure dictionaries always split, + even when shorter than the max line length. +- New knob `SPACE_INSIDE_BRACKETS` to add spaces inside brackets, braces, and + parentheses. +- New knob `SPACES_AROUND_SUBSCRIPT_COLON` to add spaces around the subscript / + slice operator. +### Changed +- Renamed "chromium" style to "yapf". Chromium will now use PEP-8 directly. +- `CONTINUATION_ALIGN_STYLE` with `FIXED` or `VALIGN-RIGHT` now works with + space indentation. +### Fixed +- Honor a disable directive at the end of a multiline comment. +- Don't require splitting before comments in a list when + `SPLIT_ALL_TOP_LEVEL_COMMA_SEPARATED_VALUES` is set. The knob is meant for + values, not comments, which may be associated with the current line. +- Don't over-indent a parameter list when not needed. But make sure it is + properly indented so that it doesn't collide with the lines afterwards. +- Don't split between two-word comparison operators: "is not", "not in", etc. + +## [0.29.0] 2019-11-28 +### Added +- Add the `--quiet` flag to suppress output. The return code is 1 if there are + changes, similarly to the `--diff` flag. +- Add the `indent_closing_brackets` option. This is the same as the + `dedent_closing_brackets` option except the brackets are indented the same + as the previous line. +### Changed +- Collect a parameter list into a single object. This allows us to track how a + parameter list is formatted, keeping state along the way. This helps when + supporting Python 3 type annotations. +- Catch and report `UnicodeDecodeError` exceptions. +- Improved description of .yapfignore syntax. +### Fixed +- Format subscript lists so that splits are essentially free after a comma. +- Don't add a space between a string and its subscript. +- Extend discovery of '.style.yapf' & 'setup.cfg' files to search the root + directory as well. +- Make sure we have parameters before we start calculating penalties for + splitting them. +- Indicate if a class/function is nested to ensure blank lines when needed. +- Fix extra indentation in async-for else statement. +- A parameter list with no elements shouldn't count as exceeding the column + limit. +- When splitting all comma separated values, don't treat the ending bracket as + special. +- The "no blank lines between nested classes or functions" knob should only + apply to the first nested class or function, not all of them. + +## [0.28.0] 2019-07-11 +### Added +- New knob `SPLIT_ALL_TOP_LEVEL_COMMA_SEPARATED_VALUES` is a variation on + `SPLIT_ALL_COMMA_SEPARATED_VALUES` in which, if a subexpression with a comma + fits in its starting line, then the subexpression is not split (thus avoiding + unnecessary splits). +### Changed +- Set `INDENT_DICTIONARY_VALUE` for Google style. +- Set `JOIN_MULTIPLE_LINES = False` for Google style. +### Fixed +- `BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF=False` wasn't honored because the + number of newlines was erroneously calculated beforehand. +- Lambda expressions shouldn't have an increased split penalty applied to the + 'lambda' keyword. This prevents them from being properly formatted when they're + arguments to functions. +- A comment with continuation markers (??) shouldn't mess with the lineno count. +- Only emit unformatted if the "disable long line" is at the end of the line. + Otherwise we could mess up formatting for containers which have them + interspersed with code. +- Fix a potential race condition by using the correct style for opening a file + which may not exist. + +## [0.27.0] 2019-04-07 +### Added +- `SPLIT_BEFORE_ARITHMETIC_OPERATOR` splits before an arithmetic operator when + set. `SPLIT_PENALTY_ARITHMETIC_OPERATOR` allows you to set the split penalty + around arithmetic operators. +### Changed +- Catch lib2to3's "TokenError" exception and output a nicer message. +### Fixed +- Parse integer lists correctly, removing quotes if the list is within a + string. +- Adjust the penalties of bitwise operands for '&' and '^', similar to '|'. +- Avoid splitting after opening parens if SPLIT_BEFORE_FIRST_ARGUMENT is set + to False. +- Adjust default SPLIT_PENALTY_AFTER_OPENING_BRACKET. +- Re-enable removal of extra lines on the boundaries of formatted regions. +- Adjust list splitting to avoid splitting before a dictionary element, because + those are likely to be split anyway. If we do split, it leads to horrible + looking code. +- Dictionary arguments were broken in a recent version. It resulted in + unreadable formatting, where the remaining arguments were indented far more + than the dictionary. Fixed so that if the dictionary is the first argument in + a function call and doesn't fit on a single line, then it forces a split. +- Improve the connectiveness between items in a list. This prevents random + splitting when it's not 100% necessary. +- Don't remove a comment attached to a previous object just because it's part + of the "prefix" of a function/class node. + +## [0.26.0] 2019-02-08 +### Added +- `ALLOW_SPLIT_BEFORE_DEFAULT_OR_NAMED_ASSIGNS` allows us to split before + default / named assignments. +- `ARITHMETIC_PRECEDENCE_INDICATION` removes spacing around binary operators + if they have higher precedence than other operators in the same expression. +### Changed +- `SPACES_BEFORE_COMMENT` can now be assigned to a specific value (standard + behavior) or a list of column values. When assigned to a list, trailing + comments will be horizontally aligned to the first column value within + the list that is greater than the maximum line length in the block. +- Don't modify the vertical spacing of a line that has a comment "pylint: + disable=line-too-long". The line is expected to be too long. +- improved `CONTINUATION_ALIGN_STYLE` to accept quoted or underline-separated + option value for passing option with command line arguments. +### Fixed +- When retrieving the opening bracket make sure that it's actually an opening + bracket. +- Don't completely deny a lambda formatting if it goes over the column limit. + Split only if absolutely necessary. +- Bump up penalty for splitting before a dot ('.'). +- Ignore pseudo tokens when calculating split penalties. +- Increase the penalty for splitting before the first bit of a subscript. +- Improve splitting before dictionary values. Look more closely to see if the + dictionary entry is a container. If so, then it's probably split over + multiple lines with the opening bracket on the same line as the key. + Therefore, we shouldn't enforce a split because of that. +- Increase split penalty around exponent operator. +- Correct spacing when using binary operators on strings with the + `NO_SPACES_AROUND_SELECTED_BINARY_OPERATORS` option enabled. + +## [0.25.0] 2018-11-25 +### Added +- Added `INDENT_BLANK_LINES` knob to select whether the blank lines are empty + or indented consistently with the current block. +- Support additional file exclude patterns in .yapfignore file. +### Fixed +- Correctly determine if a scope is the last in line. It avoids a wrong + computation of the line end when determining if it must split after the + opening bracket with `DEDENT_CLOSING_BRACKETS` enabled. + +## [0.24.0] 2018-09-07 +### Added +- Added 'SPLIT_BEFORE_DOT' knob to support "builder style" calls. The "builder + style" option didn't work as advertised. Lines would split after the dots, + not before them regardless of the penalties. +### Changed +- Support Python 3.7 in the tests. The old "comp_for" and "comp_if" nodes are + now "old_comp_for" and "old_comp_if" in lib2to3. +### Fixed +- Don't count inner function calls when marking arguments as named assignments. +- Make sure that tuples and the like are formatted nicely if they all can't fit + on a single line. This is similar to how we format function calls within an + argument list. +- Allow splitting in a subscript if it goes over the line limit. +- Increase the split penalty for an if-expression. +- Increase penalty for splitting in a subscript so that it's more likely to + split in a function call or other data literal. +- Cloning a pytree node doesn't transfer its a annotations. Make sure we do + that so that we don't lose information. +- Revert change that broke the "no_spaces_around_binary_operators" option. +- The "--style-help" option would output string lists and sets in Python types. + If the output was used as a style, then it wouldn't parse those values + correctly. + +## [0.23.0] 2018-08-27 +### Added +- `DISABLE_ENDING_COMMA_HEURISTIC` is a new knob to disable the heuristic which + splits a list onto separate lines if the list is comma-terminated. +### Fixed +- There's no need to increase N_TOKENS. In fact, it causes other things which + use lib2to3 to fail if called from YAPF. +- Change the exception message instead of creating a new one that's just a + clone. +- Make sure not to reformat when a line is disabled even if the --lines option + is specified. +- The "no spaces around operators" flag wasn't correctly converting strings to + sets. Changed the regexp to handle it better. + +## [0.22.0] 2018-05-15 +### Added +- The `BLANK_LINE_BEFORE_MODULE_DOCSTRING` knob adds a blank line before a + module's docstring. +- The `SPLIT_ALL_COMMA_SEPARATED_VALUES` knob causes all lists, tuples, dicts + function defs, etc... to split on all values, instead of maximizing the + number of elements on each line, when not able to fit on a single line. +### Changed +- Improve the heuristic we use to determine when to split at the start of a + function call. First check whether or not all elements can fit in the space + without wrapping. If not, then we split. +- Check all of the elements of a tuple. Similarly to how arguments are + analyzed. This allows tuples to be split more rationally. +- Adjust splitting penalties around arithmetic operators so that the code can + flow more freely. The code must flow! +- Try to meld an argument list's closing parenthesis to the last argument. +### Fixed +- Attempt to determine if long lambdas are allowed. This can be done on a + case-by-case basis with a "pylint" disable comment. +- A comment before a decorator isn't part of the decorator's line. +- Only force a new wrapped line after a comment in a decorator when it's the + first token in the decorator. + +## [0.21.0] 2018-03-18 +### Added +- Introduce a new option of formatting multiline literals. Add + `SPLIT_BEFORE_CLOSING_BRACKET` knob to control whether closing bracket should + get their own line. +- Added `CONTINUATION_ALIGN_STYLE` knob to choose continuation alignment style + when `USE_TABS` is enabled. +- Add 'BLANK_LINES_AROUND_TOP_LEVEL_DEFINITION' knob to control the number + of blank lines between top-level function and class definitions. +### Fixed +- Don't split ellipses. + +## [0.20.2] 2018-02-12 +### Changed +- Improve the speed at which files are excluded by ignoring them earlier. +- Allow dictionaries to stay on a single line if they only have one entry +### Fixed +- Use tabs when constructing a continuation line when `USE_TABS` is enabled. +- A dictionary entry may not end in a colon, but may be an "unpacking" + operation: `**foo`. Take that into account and don't split after the + unpacking operator. + +## [0.20.1] 2018-01-13 +### Fixed +- Don't treat 'None' as a keyword if calling a function on it, like '__ne__()'. +- use_tabs=True always uses a single tab per indentation level; spaces are + used for aligning vertically after that. +- Relax the split of a paren at the end of an if statement. With + `dedent_closing_brackets` option requires that it be able to split there. + +## [0.20.0] 2017-11-14 +### Added +- Improve splitting of comprehensions and generators. Add + `SPLIT_PENALTY_COMPREHENSION` knob to control preference for keeping + comprehensions on a single line and `SPLIT_COMPLEX_COMPREHENSION` to enable + splitting each clause of complex comprehensions onto its own line. +### Changed +- Take into account a named function argument when determining if we should + split before the first argument in a function call. +- Split before the first argument in a function call if the arguments contain a + dictionary that doesn't fit on a single line. +- Improve splitting of elements in a tuple. We want to split if there's a + function call in the tuple that doesn't fit on the line. +### Fixed +- Enforce spaces between ellipses and keywords. +- When calculating the split penalty for a "trailer", process the child nodes + afterwards because their penalties may change. For example if a list + comprehension is an argument. +- Don't enforce a split before a comment after the opening of a container if it + doesn't it on the current line. We try hard not to move such comments around. +- Use a TextIOWrapper when reading from stdin in Python3. This is necessary for + some encodings, like cp936, used on Windows. +- Remove the penalty for a split before the first argument in a function call + where the only argument is a generator expression. + +## [0.19.0] 2017-10-14 +### Added +- Added `SPLIT_BEFORE_EXPRESSION_AFTER_OPENING_PAREN` that enforces a split + after the opening paren of an expression that's surrounded by parens. +### Changed +- Split before the ending bracket of a comma-terminated tuple / argument list + if it's not a single element tuple / arg list. +### Fixed +- Prefer to split after a comma in an argument list rather than in the middle + of an argument. +- A non-multiline string may have newlines if it contains continuation markers + itself. Don't add a newline after the string when retaining the vertical + space. +- Take into account the "async" keyword when determining if we must split + before the first argument. +- Increase affinity for "atom" arguments in function calls. This helps prevent + lists from being separated when they don't need to be. +- Don't place a dictionary argument on its own line if it's the last argument + in the function call where that function is part of a builder-style call. +- Append the "var arg" type to a star in a star_expr. + +## [0.18.0] 2017-09-18 +### Added +- Option `ALLOW_SPLIT_BEFORE_DICT_VALUE` allows a split before a value. If + False, then it won't be split even if it goes over the column limit. +### Changed +- Use spaces around the '=' in a typed name argument to align with 3.6 syntax. +### Fixed +- Allow semicolons if the line is disabled. +- Fix issue where subsequent comments at decreasing levels of indentation + were improperly aligned and/or caused output with invalid syntax. +- Fix issue where specifying a line range removed a needed line before a + comment. +- Fix spacing between unary operators if one is 'not'. +- Indent the dictionary value correctly if there's a multi-line key. +- Don't remove needed spacing before a comment in a dict when in "chromium" + style. +- Increase indent for continuation line with same indent as next logical line + with 'async with' statement. + +## [0.17.0] 2017-08-20 +### Added +- Option `NO_SPACES_AROUND_SELECTED_BINARY_OPERATORS` prevents adding spaces + around selected binary operators, in accordance with the current style guide. +### Changed +- Adjust blank lines on formatting boundaries when using the `--lines` option. +- Return 1 if a diff changed the code. This is in line with how GNU diff acts. +- Add `-vv` flag to print out file names as they are processed +### Fixed +- Corrected how `DEDENT_CLOSING_BRACKETS` and `COALESCE_BRACKETS` interacted. +- Fix return value to return a boolean. +- Correct vim plugin not to clobber edited code if yapf returns an error. +- Ensured comma-terminated tuples with multiple elements are split onto separate lines. + +## [0.16.3] 2017-07-13 +### Changed +- Add filename information to a ParseError exception. +### Fixed +- A token that ends in a continuation marker may have more than one newline in + it, thus changing its "lineno" value. This can happen if multiple + continuation markers are used with no intervening tokens. Adjust the line + number to account for the lines covered by those markers. +- Make sure to split after a comment even for "pseudo" parentheses. + +## [0.16.2] 2017-05-19 +### Fixed +- Treat expansion operators ('*', '**') in a similar way to function calls to + avoid splitting directly after the opening parenthesis. +- Increase the penalty for splitting after the start of a tuple. +- Increase penalty for excess characters. +- Check that we have enough children before trying to access them all. +- Remove trailing whitespaces from comments. +- Split before a function call in a list if the full list isn't able to fit on + a single line. +- Trying not to split around the '=' of a named assign. +- Changed split before the first argument behavior to ignore compound + statements like if and while, but not function declarations. +- Changed coalesce brackets not to line split before closing bracket. + +## [0.16.1] 2017-03-22 +### Changed +- Improved performance of cloning the format decision state object. This + improved the time in one *large* case from 273.485s to 234.652s. +- Relax the requirement that a named argument needs to be on one line. Going + over the column limit is more of an issue to pylint than putting named args + on multiple lines. +- Don't make splitting penalty decisions based on the original formatting. This + can and does lead to non-stable formatting, where yapf will reformat the same + code in different ways. +### Fixed +- Ensure splitting of arguments if there's a named assign present. +- Prefer to coalesce opening brackets if it's not at the beginning of a + function call. +- Prefer not to squish all of the elements in a function call over to the + right-hand side. Split the arguments instead. +- We need to split a dictionary value if the first element is a comment anyway, + so don't force the split here. It's forced elsewhere. +- Ensure tabs are used for continued indentation when USE_TABS is True. + +## [0.16.0] 2017-02-05 +### Added +- The `EACH_DICT_ENTRY_ON_SEPARATE_LINE` knob indicates that each dictionary + entry should be in separate lines if the full dictionary isn't able to fit on + a single line. +- The `SPLIT_BEFORE_DICT_SET_GENERATOR` knob splits before the `for` part of a + dictionary/set generator. +- The `BLANK_LINE_BEFORE_CLASS_DOCSTRING` knob adds a blank line before a + class's docstring. +- The `ALLOW_MULTILINE_DICTIONARY_KEYS` knob allows dictionary keys to span + more than one line. +### Fixed +- Split before all entries in a dict/set or list maker when comma-terminated, + even if there's only one entry. +- Will now try to set O_BINARY mode on stdout under Windows and Python 2. +- Avoid unneeded newline transformation when writing formatted code to + output on (affects only Python 2) + +## [0.15.2] 2017-01-29 +### Fixed +- Don't perform a global split when a named assign is part of a function call + which itself is an argument to a function call. I.e., don't cause 'a' to + split here: + + func(a, b, c, d(x, y, z=42)) +- Allow splitting inside a subscript if it's a logical or bitwise operating. + This should keep the subscript mostly contiguous otherwise. + +## [0.15.1] 2017-01-21 +### Fixed +- Don't insert a space between a type hint and the '=' sign. +- The '@' operator can be used in Python 3 for matrix multiplication. Give the + '@' in the decorator a DECORATOR subtype to distinguish it. +- Encourage the formatter to split at the beginning of an argument list instead + of in the middle. Especially if the middle is an empty parameter list. This + adjusts the affinity of binary and comparison operators. In particular, the + "not in" and other such operators don't want to have a split after it (or + before it) if at all possible. + +## [0.15.0] 2017-01-12 +### Added +- Keep type annotations intact as much as possible. Don't try to split the over + multiple lines. +### Fixed +- When determining if each element in a dictionary can fit on a single line, we + are skipping dictionary entries. However, we need to ignore comments in our + calculations and implicitly concatenated strings, which are already placed on + separate lines. +- Allow text before a "pylint" comment. +- Also allow text before a "yapf: (disable|enable)" comment. + +## [0.14.0] 2016-11-21 +### Added +- formatting can be run in parallel using the "-p" / "--parallel" flags. +### Fixed +- "not in" and "is not" should be subtyped as binary operators. +- A non-Node dictionary value may have a comment before it. In those cases, we + want to avoid encompassing only the comment in pseudo parens. So we include + the actual value as well. +- Adjust calculation so that pseudo-parentheses don't count towards the total + line length. +- Don't count a dictionary entry as not fitting on a single line in a + dictionary. +- Don't count pseudo-parentheses in the length of the line. + +## [0.13.2] 2016-10-22 +### Fixed +- REGRESSION: A comment may have a prefix with newlines in it. When calculating + the prefix indent, we cannot take the newlines into account. Otherwise, the + comment will be misplaced causing the code to fail. + +## [0.13.1] 2016-10-17 +### Fixed +- Correct emitting a diff that was accidentally removed. + +## [0.13.0] 2016-10-16 +### Added +- Added support to retain the original line endings of the source code. + +### Fixed +- Functions or classes with comments before them were reformatting the comments + even if the code was supposed to be ignored by the formatter. We now don't + adjust the whitespace before a function's comment if the comment is a + "disabled" line. We also don't count "# yapf: {disable|enable}" as a disabled + line, which seems logical. +- It's not really more readable to split before a dictionary value if it's part + of a dictionary comprehension. +- Enforce two blank lines after a function or class definition, even before a + comment. (But not between a decorator and a comment.) This is related to PEP8 + error E305. +- Remove O(n^2) algorithm from the line disabling logic. + +## [0.12.2] 2016-10-09 +### Fixed +- If `style.SetGlobalStyle()` was called and then + `yapf_api.FormatCode` was called, the style set by the first call would be + lost, because it would return the style created by `DEFAULT_STYLE_FACTORY`, + which is set to PEP8 by default. Fix this by making the first call set which + factory we call as the "default" style. +- Don't force a split before non-function call arguments. +- A dictionary being used as an argument to a function call and which can exist + on a single line shouldn't be split. +- Don't rely upon the original line break to determine if we should split + before the elements in a container. Especially split if there's a comment in + the container. +- Don't add spaces between star and args in a lambda expression. +- If a nested data structure terminates in a comma, then split before the first + element, but only if there's more than one element in the list. + +## [0.12.1] 2016-10-02 +### Changed +- Dictionary values will be placed on the same line as the key if *all* of the + elements in the dictionary can be placed on one line. Otherwise, the + dictionary values will be placed on the next line. + +### Fixed +- Prefer to split before a terminating r-paren in an argument list if the line + would otherwise go over the column limit. +- Split before the first key in a dictionary if the dictionary cannot fit on a + single line. +- Don't count "pylint" comments when determining if the line goes over the + column limit. +- Don't count the argument list of a lambda as a named assign in a function + call. + +## [0.12.0] 2016-09-25 +### Added +- Support formatting of typed names. Typed names are formatted a similar way to + how named arguments are formatted, except that there's a space after the + colon. +- Add a knob, 'SPACES_AROUND_DEFAULT_OR_NAMED_ASSIGN', to allow adding spaces + around the assign operator on default or named assigns. + +## Changed +- Turn "verification" off by default for external APIs. +- If a function call in an argument list won't fit on the current line but will + fit on a line by itself, then split before the call so that it won't be split + up unnecessarily. + +## Fixed +- Don't add space after power operator if the next operator's a unary operator. + +## [0.11.1] 2016-08-17 +### Changed +- Issue #228: Return exit code 0 on success, regardless of whether files were + changed. (Previously, 0 meant success with no files + modified, and 2 meant success with at least one file modified.) + +### Fixed +- Enforce splitting each element in a dictionary if comma terminated. +- It's okay to split in the middle of a dotted name if the whole expression is + going to go over the column limit. +- Asynchronous functions were going missing if they were preceded by a comment + (a what? exactly). The asynchronous function processing wasn't taking the + comment into account and thus skipping the whole function. +- The splitting of arguments when comma terminated had a conflict. The split + penalty of the closing bracket was set to the maximum, but it shouldn't be if + the closing bracket is preceded by a comma. + +## [0.11.0] 2016-07-17 +### Added +- The COALESCE_BRACKETS knob prevents splitting consecutive brackets when + DEDENT_CLOSING_BRACKETS is set. +- Don't count "pylint" directives as exceeding the column limit. + +### Changed +- We split all of the arguments to a function call if there's a named argument. + In this case, we want to split after the opening bracket too. This makes + things look a bit better. + +### Fixed +- When retaining format of a multiline string with Chromium style, make sure + that the multiline string doesn't mess up where the following comma ends up. +- Correct for when 'lib2to3' smooshes comments together into the same DEDENT + node. + +## [0.10.0] 2016-06-14 +### Added +- Add a knob, 'USE_TABS', to allow using tabs for indentation. + +### Changed +- Performance enhancements. + +### Fixed +- Don't split an import list if it's not surrounded by parentheses. + +## [0.9.0] 2016-05-29 +### Added +- Added a knob (SPLIT_PENALTY_BEFORE_IF_EXPR) to adjust the split penalty + before an if expression. This allows the user to place a list comprehension + all on one line. +- Added a knob (SPLIT_BEFORE_FIRST_ARGUMENT) that encourages splitting before + the first element of a list of arguments or parameters if they are going to + be split anyway. +- Added a knob (SPLIT_ARGUMENTS_WHEN_COMMA_TERMINATED) splits arguments to a + function if the list is terminated by a comma. + +### Fixed +- Don't split before a first element list argument as we would before a first + element function call. +- Don't penalize when we must split a line. +- Allow splitting before the single argument in a function call. + +## [0.8.2] 2016-05-21 +### Fixed +- Prefer not to split after the opening of a subscript. +- Don't add space before the 'await' keyword if it's preceded by an opening + paren. +- When we're setting the split penalty for a continuous list, we don't want to + mistake a comment at the end of that list as part of the list. +- When calculating blank lines, don't assume the last seen object was a class + or function when we're in a class or function. +- Don't count the closing scope when determining if the current scope is the + last scope on the line. + +## [0.8.1] 2016-05-18 +### Fixed +- 'SPLIT_BEFORE_LOGICAL_OPERATOR' wasn't working correctly. The penalty was + being set incorrectly when it was part of a larger construct. +- Don't separate a keyword, like "await", from a left paren. +- Don't rely upon the original tokens' line number to determine if we should + perform splitting in Facebook mode. The line number isn't the line number of + the reformatted token, but the line number where it was in the original code. + Instead, we need to carefully determine if the line is liabel to be split and + act accordingly. + +## [0.8.0] 2016-05-10 +### Added +- Add a knob, 'SPACES_AROUND_POWER_OPERATOR', to allow adding spaces around the + power operator. + +### Fixed +- There shouldn't be a space between a decorator and an intervening comment. +- If we split before a bitwise operator, then we assume that the programmer + knows what they're doing, more or less, and so we enforce a split before said + operator if one exists in the original program. +- Strengthen the bond between a keyword and value argument. +- Don't add a blank line after a multiline string. +- If the "for" part of a list comprehension can exist on the starting line + without going over the column limit, then let it remain there. + +## [0.7.1] 2016-04-21 +### Fixed +- Don't rewrite the file if there are no changes. +- Ensure the proper number of blank lines before an async function. +- Split after a bitwise operator when in PEP 8 mode. +- Retain the splitting within a dictionary data literal between the key and + value. +- Try to keep short function calls all on one line even if they're part of a + larger series of tokens. This stops us from splitting too much. + +## [0.7.0] 2016-04-09 +### Added +- Support for Python 3.5. +- Add 'ALLOW_MULTILINE_LAMBDAS' which allows lambdas to be formatted onto + multiple lines. + +### Fixed +- Lessen penalty for splitting before a dictionary keyword. +- Formatting of trailing comments on disabled formatting lines. +- Disable / enable formatting at end of multi-line comment. + +## [0.6.3] 2016-03-06 +### Changed +- Documentation updated. + +### Fixed +- Fix spacing of multiline comments when formatting is disabled. + +## [0.6.2] 2015-11-01 +### Changed +- Look at the 'setup.cfg' file to see if it contains style information for + YAPF. +- Look at the '~/.config/yapf/style' file to see if it contains global style + information for YAPF. + +### Fixed +- Make lists that can fit on one line more likely to stay together. +- Correct formatting of '*args' and '**kwargs' when there are default values in + the argument list. + +## [0.6.1] 2015-10-24 +### Fixed +- Make sure to align comments in data literals correctly. Also make sure we + don't count a "#." in a string as an i18n comment. +- Retain proper vertical spacing before comments in a data literal. +- Make sure that continuations from a compound statement are distinguished from + the succeeding line. +- Ignore preceding comments when calculating what is a "dictionary maker". +- Add a small penalty for splitting before a closing bracket. +- Ensure that a space is enforced after we remove a pseudo-paren that's between + two names, keywords, numbers, etc. +- Increase the penalty for splitting after a pseudo-paren. This could lead to + less readable code in some circumstances. + +## [0.6.0] 2015-10-18 +### Added +- Add knob to indent the dictionary value if there is a split before it. + +### Changed +- No longer check that a file is a "Python" file unless the '--recursive' flag + is specified. +- No longer allow the user to specify a directory unless the '--recursive' flag + is specified. + +### Fixed +- When determining if we should split a dictionary's value to a new line, use + the longest entry instead of the total dictionary's length. This allows the + formatter to reformat the dictionary in a more consistent manner. +- Improve how list comprehensions are formatted. Make splitting dependent upon + whether the "comp_for" or "comp_if" goes over the column limit. +- Don't over indent if expression hanging indents if we expect to dedent the + closing bracket. +- Improve splitting heuristic when the first argument to a function call is + itself a function call with arguments. In cases like this, the remaining + arguments to the function call would look badly aligned, even though they are + technically correct (the best kind of correct!). +- Improve splitting heuristic more so that if the first argument to a function + call is a data literal that will go over the column limit, then we want to + split before it. +- Remove spaces around '**' operator. +- Retain formatting of comments in the middle of an expression. +- Don't add a newline to an empty file. +- Over indent a function's parameter list if it's not distinguished from the + body of the function. + +## [0.5.0] 2015-10-11 +### Added +- Add option to exclude files/directories from formatting. +- Add a knob to control whether import names are split after the first '('. + +### Fixed +- Indent the continuation of an if-then statement when it's not distinguished + from the body of the if-then. +- Allow for sensible splitting of array indices where appropriate. +- Prefer to not split before the ending bracket of an atom. This produces + better code in most cases. +- Corrected how horizontal spaces were presevered in a disabled region. + +## [0.4.0] 2015-10-07 +### Added +- Support for dedenting closing brackets, "facebook" style. + +### Fixed +- Formatting of tokens after a multiline string didn't retain their horizontal + spacing. + +## [0.3.1] 2015-09-30 +### Fixed +- Format closing scope bracket correctly when indentation size changes. + +## [0.3.0] 2015-09-20 +### Added +- Return a 2 if the source changed, 1 on error, and 0 for no change. + +### Fixed +- Make sure we format if the "lines" specified are in the middle of a + statement. + +## [0.2.9] - 2015-09-13 +### Fixed +- Formatting of multiple files. It was halting after formatting the first file. + +## [0.2.8] - 2015-09-12 +### Added +- Return a non-zero exit code if the source was changed. +- Add bitwise operator splitting penalty and prefer to split before bitwise + operators. + +### Fixed +- Retain vertical spacing between disabled and enabled lines. +- Split only at start of named assign. +- Retain comment position when formatting is disabled. +- Honor splitting before or after logical ops. diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.md similarity index 58% rename from CONTRIBUTING.rst rename to CONTRIBUTING.md index e6a71800b..4c0273cd4 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.md @@ -1,12 +1,13 @@ -Want to contribute? Great! First, read this page (including the small print at the end). +# How to Contribute -Before you contribute ---------------------- +Want to contribute? Great! First, read this page (including the small print at +the end). -Before we can use your code, you must sign the `Google Individual Contributor -License Agreement -`_ (CLA), which -you can do online. The CLA is necessary mainly because you own the +## Before you contribute + +Before we can use your code, you must sign the [Google Individual Contributor +License Agreement](https://developers.google.com/open-source/cla/individual?csw=1) +(CLA), which you can do online. The CLA is necessary mainly because you own the copyright to your changes, even after your contribution becomes part of our codebase, so we need your permission to use and distribute your code. We also need to be sure of various other things—for instance that you'll tell us if you @@ -18,27 +19,31 @@ us first through the issue tracker with your idea so that we can help out and possibly guide you. Coordinating up front makes it much easier to avoid frustration later on. -Code reviews ------------- +## Code reviews All submissions, including submissions by project members, require review. We use Github pull requests for this purpose. -YAPF coding style ------------------ +## YAPF coding style -YAPF follows the `Chromium Python Style Guide -`_. It's the same -as the Google Python Style guide with two exceptions: +YAPF follows the [Google Python Style Guide](https://google.github.io/styleguide/pyguide.html) +with two exceptions: - 2 spaces for indentation rather than 4. -- CamelCase for function and method names rather than words_with_underscores. +- CamelCase for function and method names rather than `snake_case`. + +The rationale for this is that YAPF was initially developed at Google where +these two exceptions are still part of the internal Python style guide. + +## Getting started +YAPF supports using tox 3 for creating a local dev environment, testing, and +building redistributables. See [HACKING.md](HACKING.md) for more info. -The rationale for this is that YAPF was initially developed at Google where these -two exceptions are still part of the internal Python style guide. +```bash +$ pipx run --spec='tox<4' tox --devenv .venv +``` -Small print ------------ +## Small print Contributions made by corporations are covered by a different agreement than the one above, the Software Grant and Corporate Contributor License Agreement. diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 62e2a5f21..29b874c6c 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -13,3 +13,8 @@ Bill Wendling Eli Bendersky Sam Clegg Łukasz Langa +Oleg Butuzov +Mauricio Herrera Cuadra +Xiao Wang +Andrea Oliveri +Kyle Gottfried diff --git a/EDITOR SUPPORT.md b/EDITOR SUPPORT.md new file mode 100644 index 000000000..b6f7acb2e --- /dev/null +++ b/EDITOR SUPPORT.md @@ -0,0 +1,82 @@ +# Using YAPF with your editor + +YAPF is supported by multiple editors via community extensions or plugins. + +- [IntelliJ/PyCharm](#intellijpycharm) +- [IPython](#ipython) +- [VSCode](#vscode) + +## IntelliJ/PyCharm + +Use the `File Watchers` plugin to run YAPF against a file when you perform a save. + +1. Install the [File Watchers](https://www.jetbrains.com/help/idea/using-file-watchers.html) Plugin +1. Add the following `.idea/watcherTasks.xml` to your project. If you already have this file just add the `TaskOptions` section from below. This example uses Windows and a virtual environment, modify the `program` option as appropriate. + ```xml + + + + + + + + + ``` + +## IPython + +IPython supports formatting lines automatically when you press the `` button to submit the current code block. + +Make sure that the YAPF module is available to the IPython runtime: + +```shell +pip install ipython yapf +``` + +pipx example: + +```shell +pipx install ipython +pipx inject ipython yapf +``` + +Add following to `~/.ipython/profile_default/ipython_config.py`: + +```python +c.TerminalInteractiveShell.autoformatter = 'yapf' +``` + +## VSCode + +VSCode has deprecated support for YAPF in its official Python extension [in favor of dedicated formatter extensions](https://github.com/microsoft/vscode-python/wiki/Migration-to-Python-Tools-Extensions). + +1. Install EeyoreLee's [yapf](https://marketplace.visualstudio.com/items?itemName=eeyore.yapf) extension. +1. Install the yapf package from pip. + ``` + pip install yapf + ``` +1. Add the following to VSCode's `settings.json`: + ```jsonc + "[python]": { + "editor.formatOnSaveMode": "file", + "editor.formatOnSave": true, + "editor.defaultFormatter": "eeyore.yapf" # choose this extension + }, + ``` diff --git a/HACKING.md b/HACKING.md new file mode 100644 index 000000000..1c03e8008 --- /dev/null +++ b/HACKING.md @@ -0,0 +1,75 @@ +## Running YAPF on itself + +- To run YAPF on all of YAPF: + +```bash +$ pipx run --spec=${PWD} --no-cache yapf -m -i -r yapf/ yapftests/ third_party/ +``` + +- To run YAPF on just the files changed in the current git branch: + +```bash +$ pipx run --spec=${PWD} --no-cache yapf -m -i $(git diff --name-only @{upstream}) +``` + +## Testing and building redistributables locally + +YAPF uses tox 3 to test against multiple python versions and to build redistributables. + +Tox will opportunistically use pyenv environments when available. +To configure pyenv run the following in bash: + +```bash +$ xargs -t -n1 pyenv install < .python-version +``` + +Test against all supported Python versions that are currently installed: +```bash +$ pipx run --spec='tox<4' tox +``` + +Build and test the sdist and wheel against your default Python environment. The redistributables will be in the `dist` directory. +```bash +$ pipx run --spec='tox<4' tox -e bdist_wheel -e sdist +``` + +## Releasing a new version + +1. Install all expected pyenv environements + ```bash + $ xargs -t -n1 pyenv install < .python-version + ``` + +1. Run tests against Python 3.7 - 3.11 with + ```bash + $ pipx run --spec='tox<4' tox + ``` + +1. Bump version in `yapf/_version.py`. + +1. Build and test redistributables + + ```bash + $ pipx run --spec='tox<4' tox -e bdist_wheel -e sdist + ``` + +1. Check that it looks OK. + 1. Install it onto a virtualenv, + 1. run tests, and + 1. run yapf as a tool. + +1. Push to PyPI: + + ```bash + $ pipx run twine upload dist/* + ``` + +1. Test in a clean virtualenv that 'pip install yapf' works with the new + version. + +1. Commit the version bump and add tag with: + + ```bash + $ git tag v$(VERSION_NUM) + $ git push --tags + ``` diff --git a/HACKING.rst b/HACKING.rst deleted file mode 100644 index 097b69522..000000000 --- a/HACKING.rst +++ /dev/null @@ -1,30 +0,0 @@ -Running YAPF on itself ----------------------- - -To run YAPF on all of YAPF:: - - $ PYTHONPATH=$PWD/yapf python -m yapf -i -r . - -To run YAPF on just the files changed in the curent git branch:: - - $ PYTHONPATH=$PWD/yapf python -m yapf -i $(git diff --name-only @{upstream}) - -Releasing a new version ------------------------ - -* Run tests: python setup.py test - [don't forget to run with Python 2.7 and 3.4] - -* Bump version in yapf/__init__.py - -* Build source distribution: python setup.py sdist - -* Check it looks OK, install it onto a virtualenv, run tests, run yapf as a tool - -* Push to PyPI: python setup.py sdist bdist_wheel upload - -* Test in a clean virtualenv that 'pip install yapf' works with the new version - -* Commit the version bump; add tag with git tag v; git push --tags - -TODO: discuss how to use tox to make virtualenv testing easier. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 000000000..26bd40ec3 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,4 @@ +include HACKING.md LICENSE AUTHORS CHANGELOG.md CONTRIBUTING.md CONTRIBUTORS +include .coveragerc .editorconfig .flake8 plugins/README.md +include plugins/vim/autoload/yapf.vim plugins/vim/plugin/yapf.vim pylintrc +include .style.yapf tox.ini .travis.yml .vimrc diff --git a/README.md b/README.md new file mode 100644 index 000000000..93de3388b --- /dev/null +++ b/README.md @@ -0,0 +1,1129 @@ +# YAPF + +

+PyPI Version +Build Status +Actions Status +Coverage Status +

+ + +## Introduction + +YAPF is a Python formatter based on [`clang-format`](https://clang.llvm.org/docs/ClangFormat.html) +(developed by Daniel Jasper). In essence, the algorithm takes the code and +calculates the best formatting that conforms to the configured style. It takes +away a lot of the drudgery of maintaining your code. + +The ultimate goal is that the code YAPF produces is as good as the code that a +programmer would write if they were following the style guide. + +> **Note** +> YAPF is not an official Google product (experimental or otherwise), it is +> just code that happens to be owned by Google. + + +## Installation + +To install YAPF from PyPI: + +```bash +$ pip install yapf +``` + +YAPF is still considered in "beta" stage, and the released version may change +often; therefore, the best way to keep up-to-date with the latest development +is to clone this repository or install directly from github: + +```bash +$ pip install git+https://github.com/google/yapf.git +``` + +Note that if you intend to use YAPF as a command-line tool rather than as a +library, installation is not necessary. YAPF supports being run as a directory +by the Python interpreter. If you cloned/unzipped YAPF into `DIR`, it's +possible to run: + +```bash +$ PYTHONPATH=DIR python DIR/yapf [options] ... +``` + +## Using YAPF within your favorite editor +YAPF is supported by multiple editors via community extensions or plugins. See [Editor Support](EDITOR%20SUPPORT.md) for more info. + +## Required Python versions + +YAPF supports Python 3.7+. + + +## Usage + +```console +usage: yapf [-h] [-v] [-d | -i | -q] [-r | -l START-END] [-e PATTERN] + [--style STYLE] [--style-help] [--no-local-style] [-p] [-m] [-vv] + [files ...] + +Formatter for Python code. + +positional arguments: + files reads from stdin when no files are specified. + +optional arguments: + -h, --help show this help message and exit + -v, --version show program's version number and exit + -d, --diff print the diff for the fixed source + -i, --in-place make changes to files in place + -q, --quiet output nothing and set return value + -r, --recursive run recursively over directories + -l START-END, --lines START-END + range of lines to reformat, one-based + -e PATTERN, --exclude PATTERN + patterns for files to exclude from formatting + --style STYLE specify formatting style: either a style name (for + example "pep8" or "google"), or the name of a file + with style settings. The default is pep8 unless a + .style.yapf or setup.cfg or pyproject.toml file + located in the same directory as the source or one of + its parent directories (for stdin, the current + directory is used). + --style-help show style settings and exit; this output can be saved + to .style.yapf to make your settings permanent + --no-local-style don't search for local style definition + -p, --parallel run YAPF in parallel when formatting multiple files. + -m, --print-modified print out file names of modified files + -vv, --verbose print out file names while processing +``` + + +### Return Codes + +Normally YAPF returns zero on successful program termination and non-zero +otherwise. + +If `--diff` is supplied, YAPF returns zero when no changes were necessary, +non-zero otherwise (including program error). You can use this in a CI workflow +to test that code has been YAPF-formatted. + +### Excluding files from formatting (.yapfignore or pyproject.toml) + +In addition to exclude patterns provided on commandline, YAPF looks for +additional patterns specified in a file named `.yapfignore` or `pyproject.toml` +located in the working directory from which YAPF is invoked. + +`.yapfignore`'s syntax is similar to UNIX's filename pattern matching: + +``` +* matches everything +? matches any single character +[seq] matches any character in seq +[!seq] matches any character not in seq +``` + +Note that no entry should begin with `./`. + +If you use `pyproject.toml`, exclude patterns are specified by `ignore_patterns` key +in `[tool.yapfignore]` section. For example: + +```ini +[tool.yapfignore] +ignore_patterns = [ + "temp/**/*.py", + "temp2/*.py" +] +``` + + +Formatting style +================ + +The formatting style used by YAPF is configurable and there are many "knobs" +that can be used to tune how YAPF does formatting. See the `style.py` module +for the full list. + +To control the style, run YAPF with the `--style` argument. It accepts one of +the predefined styles (e.g., `pep8` or `google`), a path to a configuration +file that specifies the desired style, or a dictionary of key/value pairs. + +The config file is a simple listing of (case-insensitive) `key = value` pairs +with a `[style]` heading. For example: + +```ini +[style] +based_on_style = pep8 +spaces_before_comment = 4 +split_before_logical_operator = true +``` + +The `based_on_style` setting determines which of the predefined styles this +custom style is based on (think of it like subclassing). Four +styles are predefined: + +- `pep8` (default) +- `google` (based off of the [Google Python Style Guide](https://github.com/google/styleguide/blob/gh-pages/pyguide.md)) +- `yapf` (for use with Google open source projects) +- `facebook` + +See `_STYLE_NAME_TO_FACTORY` in [`style.py`](https://github.com/google/yapf/blob/main/yapf/yapflib/style.py) for details. + +It's also possible to do the same on the command line with a dictionary. For +example: + +```bash +--style='{based_on_style: pep8, indent_width: 2}' +``` + +This will take the `pep8` base style and modify it to have two space +indentations. + +YAPF will search for the formatting style in the following manner: + +1. Specified on the command line +2. In the `[style]` section of a `.style.yapf` file in either the current + directory or one of its parent directories. +3. In the `[yapf]` section of a `setup.cfg` file in either the current + directory or one of its parent directories. +4. In the `[tool.yapf]` section of a `pyproject.toml` file in either the current + directory or one of its parent directories. +5. In the `[style]` section of a `~/.config/yapf/style` file in your home + directory. + +If none of those files are found, the default style PEP8 is used. + + +Example +======= + +An example of the type of formatting that YAPF can do, it will take this ugly +code: + +```python +x = { 'a':37,'b':42, + +'c':927} + +y = 'hello ''world' +z = 'hello '+'world' +a = 'hello {}'.format('world') +class foo ( object ): + def f (self ): + return 37*-+2 + def g(self, x,y=42): + return y +def f ( a ) : + return 37+-+a[42-x : y**3] +``` + +and reformat it into: + +```python +x = {'a': 37, 'b': 42, 'c': 927} + +y = 'hello ' 'world' +z = 'hello ' + 'world' +a = 'hello {}'.format('world') + + +class foo(object): + def f(self): + return 37 * -+2 + + def g(self, x, y=42): + return y + + +def f(a): + return 37 + -+a[42 - x:y**3] +``` + + +## Example as a module + +The two main APIs for calling YAPF are `FormatCode` and `FormatFile`, these +share several arguments which are described below: + +```python +>>> from yapf.yapflib.yapf_api import FormatCode # reformat a string of code + +>>> formatted_code, changed = FormatCode("f ( a = 1, b = 2 )") +>>> formatted_code +'f(a=1, b=2)\n' +>>> changed +True +``` + +A `style_config` argument: Either a style name or a path to a file that +contains formatting style settings. If None is specified, use the default style +as set in `style.DEFAULT_STYLE_FACTORY`. + +```python +>>> FormatCode("def g():\n return True", style_config='pep8')[0] +'def g():\n return True\n' +``` + +A `lines` argument: A list of tuples of lines (ints), [start, end], that we +want to format. The lines are 1-based indexed. It can be used by third-party +code (e.g., IDEs) when reformatting a snippet of code rather than a whole file. + +```python +>>> FormatCode("def g( ):\n a=1\n b = 2\n return a==b", lines=[(1, 1), (2, 3)])[0] +'def g():\n a = 1\n b = 2\n return a==b\n' +``` + +A `print_diff` (bool): Instead of returning the reformatted source, return a +diff that turns the formatted source into reformatted source. + +```diff +>>> print(FormatCode("a==b", filename="foo.py", print_diff=True)[0]) +--- foo.py (original) ++++ foo.py (reformatted) +@@ -1 +1 @@ +-a==b ++a == b +``` + +Note: the `filename` argument for `FormatCode` is what is inserted into the +diff, the default is ``. + +`FormatFile` returns reformatted code from the passed file along with its encoding: + +```python +>>> from yapf.yapflib.yapf_api import FormatFile # reformat a file + +>>> print(open("foo.py").read()) # contents of file +a==b + +>>> reformatted_code, encoding, changed = FormatFile("foo.py") +>>> formatted_code +'a == b\n' +>>> encoding +'utf-8' +>>> changed +True +``` + +The `in_place` argument saves the reformatted code back to the file: + +```python +>>> FormatFile("foo.py", in_place=True)[:2] +(None, 'utf-8') + +>>> print(open("foo.py").read()) # contents of file (now fixed) +a == b +``` + + +## Formatting diffs + +Options: + +```console +usage: yapf-diff [-h] [-i] [-p NUM] [--regex PATTERN] [--iregex PATTERN][-v] + [--style STYLE] [--binary BINARY] + +This script reads input from a unified diff and reformats all the changed +lines. This is useful to reformat all the lines touched by a specific patch. +Example usage for git/svn users: + + git diff -U0 --no-color --relative HEAD^ | yapf-diff -i + svn diff --diff-cmd=diff -x-U0 | yapf-diff -p0 -i + +It should be noted that the filename contained in the diff is used +unmodified to determine the source file to update. Users calling this script +directly should be careful to ensure that the path in the diff is correct +relative to the current working directory. + +optional arguments: + -h, --help show this help message and exit + -i, --in-place apply edits to files instead of displaying a diff + -p NUM, --prefix NUM strip the smallest prefix containing P slashes + --regex PATTERN custom pattern selecting file paths to reformat + (case sensitive, overrides -iregex) + --iregex PATTERN custom pattern selecting file paths to reformat + (case insensitive, overridden by -regex) + -v, --verbose be more verbose, ineffective without -i + --style STYLE specify formatting style: either a style name (for + example "pep8" or "google"), or the name of a file + with style settings. The default is pep8 unless a + .style.yapf or setup.cfg or pyproject.toml file + located in the same directory as the source or one of + its parent directories (for stdin, the current + directory is used). + --binary BINARY location of binary to use for YAPF +``` + +## Python features not yet supported +* Python 3.12 – [PEP 695 – Type Parameter Syntax](https://peps.python.org/pep-0695/) – [YAPF #1170](https://github.com/google/yapf/issues/1170) +* Python 3.12 – [PEP 701 – Syntactic formalization of f-strings](https://peps.python.org/pep-0701/) – [YAPF #1136](https://github.com/google/yapf/issues/1136) + +## Knobs + +#### `ALIGN_ASSIGNMENT` + +> Align assignment or augmented assignment operators. If there is a blank +> line or objects with newline entries in between, it will start new block +> alignment. For exemple: + +```python + val_first = 1 + val_second += 2 + object = { + entry1:1, + entry2:2, + entry3:3, + } + val_third = 3 +``` + +> will be formatted as follows: + +```python + val_first = 1 + val_second += 2 + object = { + entry1: 1, + entry2: 2, + entry3: 3, + } + val_third = 3 +``` + +#### `ALIGN_ASSIGNMENT_RESTART_AFTER_COMMENTS` + +> Allows to choose whether to align assignments before and after a comment +> line independently. For exemple, if off: + +```python + val_first = 1 + val_second += 2 + # comment + val_third = 3 +``` + +will be aligned as: +```python + val_first = 1 + val_second += 2 + # comment + val_third = 3 +``` + +whereas if we turn it on: +```python + val_first = 1 + val_second += 2 + # comment + val_third = 3 +``` + +#### `ALIGN_CLOSING_BRACKET_WITH_VISUAL_INDENT` + +> Align closing bracket with visual indentation. + +#### `ALLOW_MULTILINE_LAMBDAS` + +> Allow lambdas to be formatted on more than one line. + +#### `ALLOW_MULTILINE_DICTIONARY_KEYS` + +> Allow dictionary keys to exist on multiple lines. For example: + +```python + x = { + ('this is the first element of a tuple', + 'this is the second element of a tuple'): + value, + } +``` + +#### `ALLOW_SPLIT_BEFORE_DEFAULT_OR_NAMED_ASSIGNS` + +> Allow splitting before a default / named assignment in an argument list. + +#### `ALLOW_SPLIT_BEFORE_DICT_VALUE` + +> Allow splits before the dictionary value. + +#### `ARITHMETIC_PRECEDENCE_INDICATION` + +> Let spacing indicate operator precedence. For example: + +```python + a = 1 * 2 + 3 / 4 + b = 1 / 2 - 3 * 4 + c = (1 + 2) * (3 - 4) + d = (1 - 2) / (3 + 4) + e = 1 * 2 - 3 + f = 1 + 2 + 3 + 4 +``` + +> will be formatted as follows to indicate precedence: + +```python + a = 1*2 + 3/4 + b = 1/2 - 3*4 + c = (1+2) * (3-4) + d = (1-2) / (3+4) + e = 1*2 - 3 + f = 1 + 2 + 3 + 4 +``` + +#### `BLANK_LINES_AROUND_TOP_LEVEL_DEFINITION` + +> Sets the number of desired blank lines surrounding top-level function and +> class definitions. For example: + +```python + class Foo: + pass + # <------ having two blank lines here + # <------ is the default setting + class Bar: + pass +``` + +#### `BLANK_LINE_BEFORE_CLASS_DOCSTRING` + +> Insert a blank line before a class-level docstring. + +#### `BLANK_LINE_BEFORE_MODULE_DOCSTRING` + +> Insert a blank line before a module docstring. + +#### `BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF` + +> Insert a blank line before a `def` or `class` immediately nested within +> another `def` or `class`. For example: + +```python + class Foo: + # <------ this blank line + def method(): + pass +``` + +#### `BLANK_LINES_BETWEEN_TOP_LEVEL_IMPORTS_AND_VARIABLES` + +> Sets the number of desired blank lines between top-level imports and +> variable definitions. Useful for compatibility with tools like isort. + +#### `COALESCE_BRACKETS` + +> Do not split consecutive brackets. Only relevant when +> `DEDENT_CLOSING_BRACKETS` or `INDENT_CLOSING_BRACKETS` is set. For example: + +```python + call_func_that_takes_a_dict( + { + 'key1': 'value1', + 'key2': 'value2', + } + ) +``` + +> would reformat to: + +```python + call_func_that_takes_a_dict({ + 'key1': 'value1', + 'key2': 'value2', + }) +``` + +#### `COLUMN_LIMIT` + +> The column limit (or max line-length) + +#### `CONTINUATION_ALIGN_STYLE` + +> The style for continuation alignment. Possible values are: + +> - `SPACE`: Use spaces for continuation alignment. This is default +> behavior. +> - `FIXED`: Use fixed number (`CONTINUATION_INDENT_WIDTH`) of columns +> (i.e. `CONTINUATION_INDENT_WIDTH`/`INDENT_WIDTH` tabs or +> `CONTINUATION_INDENT_WIDTH` spaces) for continuation alignment. +> - `VALIGN-RIGHT`: Vertically align continuation lines to multiple of +> `INDENT_WIDTH` columns. Slightly right (one tab or a few spaces) if cannot +> vertically align continuation lines with indent characters. + +#### `CONTINUATION_INDENT_WIDTH` + +> Indent width used for line continuations. + +#### `DEDENT_CLOSING_BRACKETS` + +> Put closing brackets on a separate line, dedented, if the bracketed +> expression can't fit in a single line. Applies to all kinds of brackets, +> including function definitions and calls. For example: + +```python + config = { + 'key1': 'value1', + 'key2': 'value2', + } # <--- this bracket is dedented and on a separate line + + time_series = self.remote_client.query_entity_counters( + entity='dev3246.region1', + key='dns.query_latency_tcp', + transform=Transformation.AVERAGE(window=timedelta(seconds=60)), + start_ts=now()-timedelta(days=3), + end_ts=now(), + ) # <--- this bracket is dedented and on a separate line +``` + +#### `DISABLE_ENDING_COMMA_HEURISTIC` + +> Disable the heuristic which places each list element on a separate line if +> the list is comma-terminated. +> +> Note: The behavior of this flag changed in v0.40.3. Before, if this flag +> was true, we would split lists that contained a trailing comma or a +> comment. Now, we have a separate flag, `DISABLE_SPLIT_LIST_WITH_COMMENT`, +> that controls splitting when a list contains a comment. To get the old +> behavior, set both flags to true. More information in +> [CHANGELOG.md](CHANGELOG.md#new-disable_split_list_with_comment-flag). + +#### `DISABLE_SPLIT_LIST_WITH_COMMENT` + +> Don't put every element on a new line within a list that contains +> interstitial comments. +> +> Without this flag (default): +> +> ``` +> [ +> a, +> b, # +> c +> ] +> ``` +> +> With this flag: +> +> ``` +> [ +> a, b, # +> c +> ] +> ``` +> +> This mirrors the behavior of clang-format and is useful for forming +> "logical groups" of elements in a list. It also works in function +> declarations. + +#### `EACH_DICT_ENTRY_ON_SEPARATE_LINE` + +> Place each dictionary entry onto its own line. + +#### `FORCE_MULTILINE_DICT` + +> Respect `EACH_DICT_ENTRY_ON_SEPARATE_LINE` even if the line is shorter than +> `COLUMN_LIMIT`. + +#### `I18N_COMMENT` + +> The regex for an internationalization comment. The presence of this comment +> stops reformatting of that line, because the comments are required to be +> next to the string they translate. + +#### `I18N_FUNCTION_CALL` + +> The internationalization function call names. The presence of this function +> stops reformatting on that line, because the string it has cannot be moved +> away from the i18n comment. + +#### `INDENT_BLANK_LINES` + +> Set to `True` to prefer indented blank lines rather than empty + +#### `INDENT_CLOSING_BRACKETS` + +> Put closing brackets on a separate line, indented, if the bracketed +> expression can't fit in a single line. Applies to all kinds of brackets, +> including function definitions and calls. For example: + +```python + config = { + 'key1': 'value1', + 'key2': 'value2', + } # <--- this bracket is indented and on a separate line + + time_series = self.remote_client.query_entity_counters( + entity='dev3246.region1', + key='dns.query_latency_tcp', + transform=Transformation.AVERAGE(window=timedelta(seconds=60)), + start_ts=now()-timedelta(days=3), + end_ts=now(), + ) # <--- this bracket is indented and on a separate line +``` + +#### `INDENT_DICTIONARY_VALUE` + +> Indent the dictionary value if it cannot fit on the same line as the +> dictionary key. For example: + +```python + config = { + 'key1': + 'value1', + 'key2': value1 + + value2, + } +``` + +#### `INDENT_WIDTH` + +> The number of columns to use for indentation. + +#### `JOIN_MULTIPLE_LINES` + +> Join short lines into one line. E.g., single line `if` statements. + +#### `NO_SPACES_AROUND_SELECTED_BINARY_OPERATORS` + +> Do not include spaces around selected binary operators. For example: + +```python + 1 + 2 * 3 - 4 / 5 +``` + +> will be formatted as follows when configured with `*`, `/`: + +```python + 1 + 2*3 - 4/5 +``` + +#### `SPACE_BETWEEN_ENDING_COMMA_AND_CLOSING_BRACKET` + +> Insert a space between the ending comma and closing bracket of a list, etc. + +#### `SPACE_INSIDE_BRACKETS` + + Use spaces inside brackets, braces, and parentheses. For example: + +```python + method_call( 1 ) + my_dict[ 3 ][ 1 ][ get_index( *args, **kwargs ) ] + my_set = { 1, 2, 3 } +``` + +#### `SPACES_AROUND_DEFAULT_OR_NAMED_ASSIGN` + +> Set to `True` to prefer spaces around the assignment operator for default +> or keyword arguments. + +#### `SPACES_AROUND_DICT_DELIMITERS` + +> Adds a space after the opening '{' and before the ending '}' dict delimiters. + +```python + {1: 2} +``` + +> will be formatted as: + +```python + { 1: 2 } +``` + +#### `SPACES_AROUND_LIST_DELIMITERS` + +> Adds a space after the opening '[' and before the ending ']' list delimiters. + +```python + [1, 2] +``` + +> will be formatted as: + +```python + [ 1, 2 ] +``` + +#### `SPACES_AROUND_POWER_OPERATOR` + +> Set to `True` to prefer using spaces around `**`. + +#### `SPACES_AROUND_SUBSCRIPT_COLON` + +> Use spaces around the subscript / slice operator. For example: + +```python + my_list[1 : 10 : 2] +``` + +##### `SPACES_AROUND_TUPLE_DELIMITERS` + +> Adds a space after the opening '(' and before the ending ')' tuple delimiters. + +```python + (1, 2, 3) +``` + +> will be formatted as: + +```python + ( 1, 2, 3 ) +``` + +#### `SPACES_BEFORE_COMMENT` + +> The number of spaces required before a trailing comment. +> This can be a single value (representing the number of spaces +> before each trailing comment) or list of values (representing +> alignment column values; trailing comments within a block will +> be aligned to the first column value that is greater than the maximum +> line length within the block). + +> **Note:** Lists of values may need to be quoted in some contexts +> (eg. shells or editor config files). + +> For example, with `spaces_before_comment=5`: + +```python + 1 + 1 # Adding values +``` + +> will be formatted as: + +```python + 1 + 1 # Adding values <-- 5 spaces between the end of the statement and comment +``` + +> with `spaces_before_comment="15, 20"`: + +```python + 1 + 1 # Adding values + two + two # More adding + + longer_statement # This is a longer statement + short # This is a shorter statement + + a_very_long_statement_that_extends_beyond_the_final_column # Comment + short # This is a shorter statement +``` + +> will be formatted as: + +```python + 1 + 1 # Adding values <-- end of line comments in block aligned to col 15 + two + two # More adding + + longer_statement # This is a longer statement <-- end of line comments in block aligned to col 20 + short # This is a shorter statement + + a_very_long_statement_that_extends_beyond_the_final_column # Comment <-- the end of line comments are aligned based on the line length + short # This is a shorter statement +``` + +#### `SPLIT_ALL_COMMA_SEPARATED_VALUES` + +> If a comma separated list (`dict`, `list`, `tuple`, or function `def`) is +> on a line that is too long, split such that each element is on a separate +> line. + +#### `SPLIT_ALL_TOP_LEVEL_COMMA_SEPARATED_VALUES` + +> Variation on `SPLIT_ALL_COMMA_SEPARATED_VALUES` in which, if a +> subexpression with a comma fits in its starting line, then the +> subexpression is not split. This avoids splits like the one for +> `b` in this code: + +```python + abcdef( + aReallyLongThing: int, + b: [Int, + Int]) +``` + +> with the new knob this is split as: + +```python + abcdef( + aReallyLongThing: int, + b: [Int, Int]) +``` + +#### `SPLIT_ARGUMENTS_WHEN_COMMA_TERMINATED` + +> Split before arguments if the argument list is terminated by a comma. + +#### `SPLIT_BEFORE_ARITHMETIC_OPERATOR` + +> Set to `True` to prefer splitting before `+`, `-`, `*`, `/`, `//`, or `@` +> rather than after. + +#### `SPLIT_BEFORE_BITWISE_OPERATOR` + +> Set to `True` to prefer splitting before `&`, `|` or `^` rather than after. + +#### `SPLIT_BEFORE_CLOSING_BRACKET` + +> Split before the closing bracket if a `list` or `dict` literal doesn't fit +> on a single line. + +#### `SPLIT_BEFORE_DICT_SET_GENERATOR` + +> Split before a dictionary or set generator (`comp_for`). For example, note +> the split before the `for`: + +```python + foo = { + variable: 'Hello world, have a nice day!' + for variable in bar if variable != 42 + } +``` + +#### `SPLIT_BEFORE_DOT` + +> Split before the `.` if we need to split a longer expression: + +```python + foo = ('This is a really long string: {}, {}, {}, {}'.format(a, b, c, d)) +``` + +> would reformat to something like: + +```python + foo = ('This is a really long string: {}, {}, {}, {}' + .format(a, b, c, d)) +``` + +#### `SPLIT_BEFORE_EXPRESSION_AFTER_OPENING_PAREN` + +> Split after the opening paren which surrounds an expression if it doesn't +> fit on a single line. + +#### `SPLIT_BEFORE_FIRST_ARGUMENT` + +> If an argument / parameter list is going to be split, then split before the +> first argument. + +#### `SPLIT_BEFORE_LOGICAL_OPERATOR` + +> Set to `True` to prefer splitting before `and` or `or` rather than after. + +#### `SPLIT_BEFORE_NAMED_ASSIGNS` + +> Split named assignments onto individual lines. + +#### `SPLIT_COMPLEX_COMPREHENSION` + +> For list comprehensions and generator expressions with multiple clauses +> (e.g multiple `for` calls, `if` filter expressions) and which need to be +> reflowed, split each clause onto its own line. For example: + +```python + result = [ + a_var + b_var for a_var in xrange(1000) for b_var in xrange(1000) + if a_var % b_var] +``` + +> would reformat to something like: + +```python + result = [ + a_var + b_var + for a_var in xrange(1000) + for b_var in xrange(1000) + if a_var % b_var] +``` + +#### `SPLIT_PENALTY_AFTER_OPENING_BRACKET` + +> The penalty for splitting right after the opening bracket. + +#### `SPLIT_PENALTY_AFTER_UNARY_OPERATOR` + +> The penalty for splitting the line after a unary operator. + +#### `SPLIT_PENALTY_ARITHMETIC_OPERATOR` + +> The penalty of splitting the line around the `+`, `-`, `*`, `/`, `//`, `%`, +> and `@` operators. + +#### `SPLIT_PENALTY_BEFORE_IF_EXPR` + +> The penalty for splitting right before an `if` expression. + +#### `SPLIT_PENALTY_BITWISE_OPERATOR` + +> The penalty of splitting the line around the `&`, `|`, and `^` operators. + +#### `SPLIT_PENALTY_COMPREHENSION` + +> The penalty for splitting a list comprehension or generator expression. + +#### `SPLIT_PENALTY_EXCESS_CHARACTER` + +> The penalty for characters over the column limit. + +#### `SPLIT_PENALTY_FOR_ADDED_LINE_SPLIT` + +> The penalty incurred by adding a line split to the logical line. The more +> line splits added the higher the penalty. + +#### `SPLIT_PENALTY_IMPORT_NAMES` + +> The penalty of splitting a list of `import as` names. For example: + +```python + from a_very_long_or_indented_module_name_yada_yad import (long_argument_1, + long_argument_2, + long_argument_3) +``` + +> would reformat to something like: + +```python + from a_very_long_or_indented_module_name_yada_yad import ( + long_argument_1, long_argument_2, long_argument_3) +``` + +#### `SPLIT_PENALTY_LOGICAL_OPERATOR` + +> The penalty of splitting the line around the `and` and `or` operators. + +#### `USE_TABS` + +> Use the Tab character for indentation. + + +## (Potentially) Frequently Asked Questions + +### Why does YAPF destroy my awesome formatting? + +YAPF tries very hard to get the formatting correct. But for some code, it won't +be as good as hand-formatting. In particular, large data literals may become +horribly disfigured under YAPF. + +The reasons for this are manyfold. In short, YAPF is simply a tool to help +with development. It will format things to coincide with the style guide, but +that may not equate with readability. + +What can be done to alleviate this situation is to indicate regions YAPF should +ignore when reformatting something: + +```python +# yapf: disable +FOO = { + # ... some very large, complex data literal. +} + +BAR = [ + # ... another large data literal. +] +# yapf: enable +``` + +You can also disable formatting for a single literal like this: + +```python +BAZ = { + (1, 2, 3, 4), + (5, 6, 7, 8), + (9, 10, 11, 12), +} # yapf: disable +``` + +To preserve the nice dedented closing brackets, use the +`dedent_closing_brackets` in your style. Note that in this case all +brackets, including function definitions and calls, are going to use +that style. This provides consistency across the formatted codebase. + +### Why Not Improve Existing Tools? + +We wanted to use clang-format's reformatting algorithm. It's very powerful and +designed to come up with the best formatting possible. Existing tools were +created with different goals in mind, and would require extensive modifications +to convert to using clang-format's algorithm. + +### Can I Use YAPF In My Program? + +Please do! YAPF was designed to be used as a library as well as a command line +tool. This means that a tool or IDE plugin is free to use YAPF. + +### I still get non-PEP8 compliant code! Why? + +YAPF tries very hard to be fully PEP 8 compliant. However, it is paramount +to not risk altering the semantics of your code. Thus, YAPF tries to be as +safe as possible and does not change the token stream +(e.g., by adding parentheses). +All these cases however, can be easily fixed manually. For instance, + +```python +from my_package import my_function_1, my_function_2, my_function_3, my_function_4, my_function_5 + +FOO = my_variable_1 + my_variable_2 + my_variable_3 + my_variable_4 + my_variable_5 + my_variable_6 + my_variable_7 + my_variable_8 +``` + +won't be split, but you can easily get it right by just adding parentheses: + +```python +from my_package import (my_function_1, my_function_2, my_function_3, + my_function_4, my_function_5) + +FOO = (my_variable_1 + my_variable_2 + my_variable_3 + my_variable_4 + + my_variable_5 + my_variable_6 + my_variable_7 + my_variable_8) +``` + + +## Gory Details + +### Algorithm Design + +The main data structure in YAPF is the `LogicalLine` object. It holds a list +of `FormatToken`\s, that we would want to place on a single line if there +were no column limit. An exception being a comment in the middle of an +expression statement will force the line to be formatted on more than one line. +The formatter works on one `LogicalLine` object at a time. + +An `LogicalLine` typically won't affect the formatting of lines before or +after it. There is a part of the algorithm that may join two or more +`LogicalLine`\s into one line. For instance, an if-then statement with a +short body can be placed on a single line: + +```python +if a == 42: continue +``` + +YAPF's formatting algorithm creates a weighted tree that acts as the solution +space for the algorithm. Each node in the tree represents the result of a +formatting decision --- i.e., whether to split or not to split before a token. +Each formatting decision has a cost associated with it. Therefore, the cost is +realized on the edge between two nodes. (In reality, the weighted tree doesn't +have separate edge objects, so the cost resides on the nodes themselves.) + +For example, take the following Python code snippet. For the sake of this +example, assume that line (1) violates the column limit restriction and needs to +be reformatted. + +```python +def xxxxxxxxxxx(aaaaaaaaaaaa, bbbbbbbbb, cccccccc, dddddddd, eeeeee): # 1 + pass # 2 +``` + +For line (1), the algorithm will build a tree where each node (a +`FormattingDecisionState` object) is the state of the line at that token given +the decision to split before the token or not. Note: the `FormatDecisionState` +objects are copied by value so each node in the graph is unique and a change in +one doesn't affect other nodes. + +Heuristics are used to determine the costs of splitting or not splitting. +Because a node holds the state of the tree up to a token's insertion, it can +easily determine if a splitting decision will violate one of the style +requirements. For instance, the heuristic is able to apply an extra penalty to +the edge when not splitting between the previous token and the one being added. + +There are some instances where we will never want to split the line, because +doing so will always be detrimental (i.e., it will require a backslash-newline, +which is very rarely desirable). For line (1), we will never want to split the +first three tokens: `def`, `xxxxxxxxxxx`, and `(`. Nor will we want to +split between the `)` and the `:` at the end. These regions are said to be +"unbreakable." This is reflected in the tree by there not being a "split" +decision (left hand branch) within the unbreakable region. + +Now that we have the tree, we determine what the "best" formatting is by finding +the path through the tree with the lowest cost. + +And that's it! diff --git a/README.rst b/README.rst deleted file mode 100644 index d86ce21d9..000000000 --- a/README.rst +++ /dev/null @@ -1,562 +0,0 @@ -==== -YAPF -==== - -.. image:: https://badge.fury.io/py/yapf.svg - :target: http://badge.fury.io/py/yapf - :alt: PyPI version - -.. image:: https://travis-ci.org/google/yapf.svg?branch=master - :target: https://travis-ci.org/google/yapf - :alt: Build status - -.. image:: https://coveralls.io/repos/google/yapf/badge.svg?branch=master - :target: https://coveralls.io/r/google/yapf?branch=master - :alt: Coverage status - - -Introduction -============ - -Most of the current formatters for Python --- e.g., autopep8, and pep8ify --- -are made to remove lint errors from code. This has some obvious limitations. -For instance, code that conforms to the PEP 8 guidelines may not be -reformatted. But it doesn't mean that the code looks good. - -YAPF takes a different approach. It's based off of 'clang-format', developed by -Daniel Jasper. In essence, the algorithm takes the code and reformats it to the -best formatting that conforms to the style guide, even if the original code -didn't violate the style guide. The idea is also similar to the 'gofmt' tool for -the Go programming language: end all holy wars about formatting - if the whole -code base of a project is simply piped through YAPF whenever modifications are -made, the style remains consistent throughout the project and there's no point -arguing about style in every code review. - -The ultimate goal is that the code YAPF produces is as good as the code that a -programmer would write if they were following the style guide. It takes away -some of the drudgery of maintaining your code. - -.. footer:: - - YAPF is not an official Google product (experimental or otherwise), it is - just code that happens to be owned by Google. - -.. contents:: - - -Installation -============ - -To install YAPF from PyPI: - -.. code-block:: - - $ pip install yapf - -YAPF is still considered in "alpha" stage, and the released version may change -often; therefore, the best way to keep up-to-date with the latest development -is to clone this repository. - -Note that if you intend to use YAPF as a command-line tool rather than as a -library, installation is not necessary. YAPF supports being run as a directory -by the Python interpreter. If you cloned/unzipped YAPF into ``DIR``, it's -possible to run: - -.. code-block:: - - $ PYTHONPATH=DIR python DIR/yapf [options] ... - - -Python versions -=============== - -YAPF supports Python 2.7 and 3.4.1+. - -YAPF requires the code it formats to be valid Python for the version YAPF itself -runs under. Therefore, if you format Python 3 code with YAPF, run YAPF itself -under Python 3 (and similarly for Python 2). - - -Usage -===== - -Options:: - - usage: yapf [-h] [-v] [-d | -i] [-r | -l START-END] [-e PATTERN] - [--style STYLE] [--style-help] [--no-local-style] - [files [files ...]] - - Formatter for Python code. - - positional arguments: - files - - optional arguments: - -h, --help show this help message and exit - -v, --version show version number and exit - -d, --diff print the diff for the fixed source - -i, --in-place make changes to files in place - -r, --recursive run recursively over directories - -l START-END, --lines START-END - range of lines to reformat, one-based - -e PATTERN, --exclude PATTERN - patterns for files to exclude from formatting - --style STYLE specify formatting style: either a style name (for - example "pep8" or "google"), or the name of a file - with style settings. The default is pep8 unless a - .style.yapf or setup.cfg file located in one of the - parent directories of the source file (or current - directory for stdin) - --style-help show style settings and exit - --no-local-style don't search for local style definition (.style.yapf) - - -Formatting style -================ - -The formatting style used by YAPF is configurable and there are many "knobs" -that can be used to tune how YAPF does formatting. See the ``style.py`` module -for the full list. - -To control the style, run YAPF with the ``--style`` argument. It accepts one of -the predefined styles (e.g., ``pep8`` or ``google``), a path to a configuration -file that specifies the desired style, or a dictionary of key/value pairs. - -The config file is a simple listing of (case-insensitive) ``key = value`` pairs -with a ``[style]`` heading. For example: - -.. code-block:: - - [style] - based_on_style = pep8 - spaces_before_comment = 4 - split_before_logical_operator = true - -The ``based_on_style`` setting determines which of the predefined styles this -custom style is based on (think of it like subclassing). - -It's also possible to do the same on the command line with a dictionary. For -example: - -.. code-block:: - - --style='{based_on_style: chromium, indent_width: 4}' - -This will take the ``chromium`` base style and modify it to have four space -indentations. - -YAPF will search for the formatting style in the following manner: - -1. Specified on the command line -2. In the `[style]` section of a `.style.yapf` file in either the current - directory or one of its parent directories. -3. In the `[yapf]` section of a `setup.cfg` file in either the current - directory or one of its parent directories. -4. In the `~/.config/yapf/style` file in your home directory. - -If none of those files are found, the default style is used (PEP8). - - -Example -======= - -An example of the type of formatting that YAPF can do, it will take this ugly -code: - -.. code-block:: python - - x = { 'a':37,'b':42, - - 'c':927} - - y = 'hello ''world' - z = 'hello '+'world' - a = 'hello {}'.format('world') - class foo ( object ): - def f (self ): - return 37*-+2 - def g(self, x,y=42): - return y - def f ( a ) : - return 37+-+a[42-x : y**3] - -and reformat it into: - -.. code-block:: python - - x = {'a': 37, 'b': 42, 'c': 927} - - y = 'hello ' 'world' - z = 'hello ' + 'world' - a = 'hello {}'.format('world') - - - class foo(object): - def f(self): - return 37 * -+2 - - def g(self, x, y=42): - return y - - - def f(a): - return 37 + -+a[42 - x:y**3] - - -Example as a module -=================== - -The two main APIs for calling yapf are ``FormatCode`` and ``FormatFile``, these -share several arguments which are described below: - -.. code-block:: python - - >>> from yapf.yapf_api import FormatCode # reformat a string of code - - >>> FormatCode("f ( a = 1, b = 2 )") - 'f(a=1, b=2)\n' - -A ``style_config`` argument: Either a style name or a path to a file that contains -formatting style settings. If None is specified, use the default style -as set in ``style.DEFAULT_STYLE_FACTORY``. - -.. code-block:: python - - >>> FormatCode("def g():\n return True", style_config='pep8') - 'def g():\n return True\n' - -A ``lines`` argument: A list of tuples of lines (ints), [start, end], -that we want to format. The lines are 1-based indexed. It can be used by -third-party code (e.g., IDEs) when reformatting a snippet of code rather -than a whole file. - -.. code-block:: python - - >>> FormatCode("def g( ):\n a=1\n b = 2\n return a==b", lines=[(1, 1), (2, 3)]) - 'def g():\n a = 1\n b = 2\n return a==b\n' - -A ``print_diff`` (bool): Instead of returning the reformatted source, return a -diff that turns the formatted source into reformatter source. - -.. code-block:: python - - >>> print(FormatCode("a==b", filename="foo.py", print_diff=True)) - --- foo.py (original) - +++ foo.py (reformatted) - @@ -1 +1 @@ - -a==b - +a == b - -Note: the ``filename`` argument for ``FormatCode`` is what is inserted into -the diff, the default is ````. - -``FormatFile`` returns reformatted code from the passed file along with its encoding: - -.. code-block:: python - - >>> from yapf.yapf_api import FormatFile # reformat a file - - >>> print(open("foo.py").read()) # contents of file - a==b - - >>> FormatFile("foo.py") - ('a == b\n', 'utf-8') - -The ``in-place`` argument saves the reformatted code back to the file: - -.. code-block:: python - - >>> FormatFile("foo.py", in_place=True) - (None, 'utf-8') - - >>> print(open("foo.py").read()) # contents of file (now fixed) - a == b - - -Knobs -===== - -``ALIGN_CLOSING_BRACKET_WITH_VISUAL_INDENT`` - Align closing bracket with visual indentation. - -``ALLOW_MULTILINE_LAMBDAS`` - Allow lambdas to be formatted on more than one line. - -``BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF`` - Insert a blank line before a ``def`` or ``class`` immediately nested within - another ``def`` or ``class``. For example: - - .. code-block:: python - - class Foo: - # <------ this blank line - def method(): - pass - -``COALESCE_BRACKETS`` - Do not split consecutive brackets. Only relevant when - ``DEDENT_CLOSING_BRACKETS`` is set. For example: - - .. code-block:: python - - call_func_that_takes_a_dict( - { - 'key1': 'value1', - 'key2': 'value2', - } - ) - - would reformat to: - - .. code-block:: python - - call_func_that_takes_a_dict({ - 'key1': 'value1', - 'key2': 'value2', - }) - - -``COLUMN_LIMIT`` - The column limit (or max line-length) - -``CONTINUATION_INDENT_WIDTH`` - Indent width used for line continuations. - -``DEDENT_CLOSING_BRACKETS`` - Put closing brackets on a separate line, dedented, if the bracketed - expression can't fit in a single line. Applies to all kinds of brackets, - including function definitions and calls. For example: - - .. code-block:: python - - config = { - 'key1': 'value1', - 'key2': 'value2', - } # <--- this bracket is dedented and on a separate line - - time_series = self.remote_client.query_entity_counters( - entity='dev3246.region1', - key='dns.query_latency_tcp', - transform=Transformation.AVERAGE(window=timedelta(seconds=60)), - start_ts=now()-timedelta(days=3), - end_ts=now(), - ) # <--- this bracket is dedented and on a separate line - -``I18N_COMMENT`` - The regex for an internationalization comment. The presence of this comment - stops reformatting of that line, because the comments are required to be - next to the string they translate. - -``I18N_FUNCTION_CALL`` - The internationalization function call names. The presence of this function - stops reformattting on that line, because the string it has cannot be moved - away from the i18n comment. - -``INDENT_DICTIONARY_VALUE`` - Indent the dictionary value if it cannot fit on the same line as the - dictionary key. For example: - - .. code-block:: python - - config = { - 'key1': - 'value1', - 'key2': value1 + - value2, - } - -``INDENT_WIDTH`` - The number of columns to use for indentation. - -``JOIN_MULTIPLE_LINES`` - Join short lines into one line. E.g., single line ``if`` statements. - -``SPACES_AROUND_POWER_OPERATOR`` - Set to ``True`` to prefer using spaces around ``**``. - -``SPACES_BEFORE_COMMENT`` - The number of spaces required before a trailing comment. - -``SPACE_BETWEEN_ENDING_COMMA_AND_CLOSING_BRACKET`` - Insert a space between the ending comma and closing bracket of a list, etc. - -``SPLIT_ARGUMENTS_WHEN_COMMA_TERMINATED`` - Split before arguments if the argument list is terminated by a comma. - -``SPLIT_BEFORE_BITWISE_OPERATOR`` - Set to ``True`` to prefer splitting before ``&``, ``|`` or ``^`` rather - than after. - -``SPLIT_BEFORE_FIRST_ARGUMENT`` - If an argument / parameter list is going to be split, then split before the - first argument. - -``SPLIT_BEFORE_LOGICAL_OPERATOR`` - Set to ``True`` to prefer splitting before ``and`` or ``or`` rather than - after. - -``SPLIT_BEFORE_NAMED_ASSIGNS`` - Split named assignments onto individual lines. - -``SPLIT_PENALTY_AFTER_OPENING_BRACKET`` - The penalty for splitting right after the opening bracket. - -``SPLIT_PENALTY_AFTER_UNARY_OPERATOR`` - The penalty for splitting the line after a unary operator. - -``SPLIT_PENALTY_BEFORE_IF_EXPR`` - The penalty for splitting right before an ``if`` expression. - -``SPLIT_PENALTY_BITWISE_OPERATOR`` - The penalty of splitting the line around the ``&``, ``|``, and ``^`` - operators. - -``SPLIT_PENALTY_EXCESS_CHARACTER`` - The penalty for characters over the column limit. - -``SPLIT_PENALTY_FOR_ADDED_LINE_SPLIT`` - The penalty incurred by adding a line split to the unwrapped line. The more - line splits added the higher the penalty. - -``SPLIT_PENALTY_IMPORT_NAMES`` - The penalty of splitting a list of ``import as`` names. For example: - - .. code-block:: python - - from a_very_long_or_indented_module_name_yada_yad import (long_argument_1, - long_argument_2, - long_argument_3) - - would reformat to something like: - - .. code-block:: python - - from a_very_long_or_indented_module_name_yada_yad import ( - long_argument_1, long_argument_2, long_argument_3) - -``SPLIT_PENALTY_LOGICAL_OPERATOR`` - The penalty of splitting the line around the ``and`` and ``or`` operators. - -``USE_TABS`` - Use the Tab character for indentation. - -(Potentially) Frequently Asked Questions -======================================== - -Why does YAPF destroy my awesome formatting? --------------------------------------------- - -YAPF tries very hard to get the formatting correct. But for some code, it won't -be as good as hand-formatting. In particular, large data literals may become -horribly disfigured under YAPF. - -The reason for this is many-fold. But in essence YAPF is simply a tool to help -with development. It will format things to coincide with the style guide, but -that may not equate with readability. - -What can be done to alleviate this situation is to indicate regions YAPF should -ignore when reformatting something: - -.. code-block:: python - - # yapf: disable - FOO = { - # ... some very large, complex data literal. - } - - BAR = [ - # ... another large data literal. - ] - # yapf: enable - -You can also disable formatting for a single literal like this: - -.. code-block:: python - - BAZ = { - (1, 2, 3, 4), - (5, 6, 7, 8), - (9, 10, 11, 12), - } # yapf: disable - -To preserve the nice dedented closing brackets, use the -``dedent_closing_brackets`` in your style. Note that in this case all -brackets, including function definitions and calls, are going to use -that style. This provides consistency across the formatted codebase. - -Why Not Improve Existing Tools? -------------------------------- - -We wanted to use clang-format's reformatting algorithm. It's very powerful and -designed to come up with the best formatting possible. Existing tools were -created with different goals in mind, and would require extensive modifications -to convert to using clang-format's algorithm. - -Can I Use YAPF In My Program? ------------------------------ - -Please do! YAPF was designed to be used as a library as well as a command line -tool. This means that a tool or IDE plugin is free to use YAPF. - - -Gory Details -============ - -Algorithm Design ----------------- - -The main data structure in YAPF is the ``UnwrappedLine`` object. It holds a list -of ``FormatToken``\s, that we would want to place on a single line if there were -no column limit. An exception being a comment in the middle of an expression -statement will force the line to be formatted on more than one line. The -formatter works on one ``UnwrappedLine`` object at a time. - -An ``UnwrappedLine`` typically won't affect the formatting of lines before or -after it. There is a part of the algorithm that may join two or more -``UnwrappedLine``\s into one line. For instance, an if-then statement with a -short body can be placed on a single line: - -.. code-block:: python - - if a == 42: continue - -YAPF's formatting algorithm creates a weighted tree that acts as the solution -space for the algorithm. Each node in the tree represents the result of a -formatting decision --- i.e., whether to split or not to split before a token. -Each formatting decision has a cost associated with it. Therefore, the cost is -realized on the edge between two nodes. (In reality, the weighted tree doesn't -have separate edge objects, so the cost resides on the nodes themselves.) - -For example, take the following Python code snippet. For the sake of this -example, assume that line (1) violates the column limit restriction and needs to -be reformatted. - -.. code-block:: python - - def xxxxxxxxxxx(aaaaaaaaaaaa, bbbbbbbbb, cccccccc, dddddddd, eeeeee): # 1 - pass # 2 - -For line (1), the algorithm will build a tree where each node (a -``FormattingDecisionState`` object) is the state of the line at that token given -the decision to split before the token or not. Note: the ``FormatDecisionState`` -objects are copied by value so each node in the graph is unique and a change in -one doesn't affect other nodes. - -Heuristics are used to determine the costs of splitting or not splitting. -Because a node holds the state of the tree up to a token's insertion, it can -easily determine if a splitting decision will violate one of the style -requirements. For instance, the heuristic is able to apply an extra penalty to -the edge when not splitting between the previous token and the one being added. - -There are some instances where we will never want to split the line, because -doing so will always be detrimental (i.e., it will require a backslash-newline, -which is very rarely desirable). For line (1), we will never want to split the -first three tokens: ``def``, ``xxxxxxxxxxx``, and ``(``. Nor will we want to -split between the ``)`` and the ``:`` at the end. These regions are said to be -"unbreakable." This is reflected in the tree by there not being a "split" -decision (left hand branch) within the unbreakable region. - -Now that we have the tree, we determine what the "best" formatting is by finding -the path through the tree with the lowest cost. - -And that's it! diff --git a/plugins/README.md b/plugins/README.md new file mode 100644 index 000000000..d2b1aaa1f --- /dev/null +++ b/plugins/README.md @@ -0,0 +1,106 @@ +# IDE Plugins + +## Emacs + +The `Emacs` plugin is maintained separately. Installation directions can be +found here: https://github.com/paetzke/py-yapf.el + + +## Vim + +The `vim` plugin allows you to reformat a range of code. Copy `plugin` and +`autoload` directories into your `~/.vim` or use `:packadd` in Vim 8. Or use +a plugin manager like Plug or Vundle: + +```vim +" Plug +Plug 'google/yapf', { 'rtp': 'plugins/vim', 'for': 'python' } + +" Vundle +Plugin 'google/yapf', { 'rtp': 'plugins/vim' } +``` + +You can add key bindings in the `.vimrc` file: + +```vim +map :call yapf#YAPF() +imap :call yapf#YAPF() +``` + +Alternatively, you can call the command `YAPF`. If you omit the range, it will +reformat the whole buffer. + +example: + +```vim +:YAPF " formats whole buffer +:'<,'>YAPF " formats lines selected in visual mode +``` + + +## Sublime Text + +The `Sublime Text` plugin is also maintained separately. It is compatible with +both Sublime Text 2 and 3. + +The plugin can be easily installed by using *Sublime Package Control*. Check +the project page of the plugin for more information: https://github.com/jason-kane/PyYapf + + +## git Pre-Commit Hook + +The `git` pre-commit hook automatically formats your Python files before they +are committed to your local repository. Any changes `yapf` makes to the files +will stay unstaged so that you can diff them manually. + +To install, simply download the raw file and copy it into your git hooks +directory: + +```bash +# From the root of your git project. +$ curl -o pre-commit.sh https://raw.githubusercontent.com/google/yapf/main/plugins/pre-commit.sh +$ chmod a+x pre-commit.sh +$ mv pre-commit.sh .git/hooks/pre-commit +``` + + +## Textmate 2 + +Plugin for `Textmate 2` requires `yapf` Python package installed on your +system: + +```bash +$ pip install yapf +``` + +Also, you will need to activate `Python` bundle from `Preferences > Bundles`. + +Finally, create a `~/Library/Application Support/TextMate/Bundles/Python.tmbundle/Commands/YAPF.tmCommand` +file with the following content: + +```xml + + + + + beforeRunningCommand + saveActiveFile + command + #!/bin/bash + +TPY=${TM_PYTHON:-python} + +"$TPY" "/usr/local/bin/yapf" "$TM_FILEPATH" + input + document + name + YAPF + scope + source.python + uuid + 297D5A82-2616-4950-9905-BD2D1C94D2D4 + + +``` + +You will see a new menu item `Bundles > Python > YAPF`. diff --git a/plugins/README.rst b/plugins/README.rst deleted file mode 100644 index a9f63dd40..000000000 --- a/plugins/README.rst +++ /dev/null @@ -1,30 +0,0 @@ -=========== -IDE Plugins -=========== - -Emacs -===== - -The ``Emacs`` plugin is maintained separately. -Installation directions can be found here: https://github.com/paetzke/py-yapf.el - -VIM -=== - -The ``vim`` plugin allows you to reformat a range of code. Place it into the -``.vim/autoload`` directory. You can add key bindings in the ``.vimrc`` file: - -.. code-block:: vim - - map :call yapf#YAPF() - imap :call yapf#YAPF() - -Sublime Text -============ - -The ``Sublime Text`` plugin is also maintained separately. -It is compatible with both Sublime Text 2 and 3. - -The plugin can be easily installed by using *Sublime Package Control*. -Check the project page of the plugin for more information: -https://github.com/jason-kane/PyYapf diff --git a/plugins/pre-commit.sh b/plugins/pre-commit.sh new file mode 100755 index 000000000..3e6873949 --- /dev/null +++ b/plugins/pre-commit.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash + +# Git pre-commit hook to check staged Python files for formatting issues with +# yapf. +# +# INSTALLING: Copy this script into `.git/hooks/pre-commit`, and mark it as +# executable. +# +# This requires that yapf is installed and runnable in the environment running +# the pre-commit hook. +# +# When running, this first checks for unstaged changes to staged files, and if +# there are any, it will exit with an error. Files with unstaged changes will be +# printed. +# +# If all staged files have no unstaged changes, it will run yapf against them, +# leaving the formatting changes unstaged. Changed files will be printed. +# +# BUGS: This does not leave staged changes alone when used with the -a flag to +# git commit, due to the fact that git stages ALL unstaged files when that flag +# is used. + +# Find all staged Python files, and exit early if there aren't any. +PYTHON_FILES=() +while IFS=$'\n' read -r line; do PYTHON_FILES+=("$line"); done \ + < <(git diff --name-only --cached --diff-filter=AM | grep --color=never '.py$') +if [ ${#PYTHON_FILES[@]} -eq 0 ]; then + exit 0 +fi + +########## PIP VERSION ############# +# Verify that yapf is installed; if not, warn and exit. +if ! command -v yapf >/dev/null; then + echo 'yapf not on path; can not format. Please install yapf:' + echo ' pip install yapf' + exit 2 +fi +######### END PIP VERSION ########## + +########## PIPENV VERSION ########## +# if ! pipenv run yapf --version 2>/dev/null 2>&1; then +# echo 'yapf not on path; can not format. Please install yapf:' +# echo ' pipenv install yapf' +# exit 2 +# fi +###### END PIPENV VERSION ########## + + +# Check for unstaged changes to files in the index. +CHANGED_FILES=() +while IFS=$'\n' read -r line; do CHANGED_FILES+=("$line"); done \ + < <(git diff --name-only "${PYTHON_FILES[@]}") +if [ ${#CHANGED_FILES[@]} -gt 0 ]; then + echo 'You have unstaged changes to some files in your commit; skipping ' + echo 'auto-format. Please stage, stash, or revert these changes. You may ' + echo 'find `git stash -k` helpful here.' + echo 'Files with unstaged changes:' "${CHANGED_FILES[@]}" + exit 1 +fi + +# Format all staged files, then exit with an error code if any have uncommitted +# changes. +echo 'Formatting staged Python files . . .' + +########## PIP VERSION ############# +yapf -i -r "${PYTHON_FILES[@]}" +######### END PIP VERSION ########## + +########## PIPENV VERSION ########## +# pipenv run yapf -i -r "${PYTHON_FILES[@]}" +###### END PIPENV VERSION ########## + + +CHANGED_FILES=() +while IFS=$'\n' read -r line; do CHANGED_FILES+=("$line"); done \ + < <(git diff --name-only "${PYTHON_FILES[@]}") +if [ ${#CHANGED_FILES[@]} -gt 0 ]; then + echo 'Reformatted staged files. Please review and stage the changes.' + echo 'Files updated: ' "${CHANGED_FILES[@]}" + exit 1 +else + exit 0 +fi diff --git a/plugins/yapf.vim b/plugins/vim/autoload/yapf.vim similarity index 70% rename from plugins/yapf.vim rename to plugins/vim/autoload/yapf.vim index 24b351070..ad3e7fc90 100644 --- a/plugins/yapf.vim +++ b/plugins/vim/autoload/yapf.vim @@ -17,8 +17,8 @@ " Place this script in your ~/.vim/autoload directory. You can add accessors to " ~/.vimrc, e.g.: " -" map :call yapf#YAPF() -" imap :call yapf#YAPF() +" map :call yapf#YAPF() +" imap :call yapf#YAPF() " function! yapf#YAPF() range " Determine range to format. @@ -26,11 +26,23 @@ function! yapf#YAPF() range let l:cmd = 'yapf --lines=' . l:line_ranges " Call YAPF with the current buffer - let l:formatted_text = system(l:cmd, join(getline(1, '$'), "\n") . "\n") + if exists('*systemlist') + let l:formatted_text = systemlist(l:cmd, join(getline(1, '$'), "\n") . "\n") + else + let l:formatted_text = + \ split(system(l:cmd, join(getline(1, '$'), "\n") . "\n"), "\n") + endif + + if v:shell_error + echohl ErrorMsg + echomsg printf('"%s" returned error: %s', l:cmd, l:formatted_text[-1]) + echohl None + return + endif " Update the buffer. execute '1,' . string(line('$')) . 'delete' - call setline(1, split(l:formatted_text, "\n")) + call setline(1, l:formatted_text) " Reset cursor to first line of the formatted range. call cursor(a:firstline, 1) diff --git a/plugins/vim/plugin/yapf.vim b/plugins/vim/plugin/yapf.vim new file mode 100644 index 000000000..e2cc7fd2c --- /dev/null +++ b/plugins/vim/plugin/yapf.vim @@ -0,0 +1,25 @@ +" Copyright 2015 Google Inc. All Rights Reserved. +" +" Licensed under the Apache License, Version 2.0 (the "License"); +" you may not use this file except in compliance with the License. +" You may obtain a copy of the License at +" +" http://www.apache.org/licenses/LICENSE-2.0 +" +" Unless required by applicable law or agreed to in writing, software +" distributed under the License is distributed on an "AS IS" BASIS, +" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +" See the License for the specific language governing permissions and +" limitations under the License. +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" +" VIM command for YAPF support +" +" Place this script in your ~/.vim/plugin directory. You can call the +" command YAPF. If you omit the range, it will reformat the whole +" buffer. +" +" example: +" :YAPF " formats whole buffer +" :'<,'>YAPF " formats lines selected in visual mode + +command! -range=% YAPF ,call yapf#YAPF() diff --git a/pylintrc b/pylintrc index 9e8a554b2..0008df24a 100644 --- a/pylintrc +++ b/pylintrc @@ -55,7 +55,7 @@ confidence= # can either give multiple identifiers separated by comma (,) or put this # option multiple times (only on the command line, not in the configuration # file where it should appear only once).You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if +# disable everything first and then re-enable specific checks. For example, if # you want to run only the similarities checker, you can use "--disable=all # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..0b4bde36c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,63 @@ +[build-system] +requires = ["setuptools>=58.5.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "yapf" +description = "A formatter for Python code" +authors = [{ name = "Google Inc." }] +maintainers = [{ name = "Bill Wendling", email = "morbo@google.com" }] +dynamic = ["version"] +license = { file = "LICENSE" } +readme = "README.md" +requires-python = ">=3.7" +classifiers = [ + 'Development Status :: 4 - Beta', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3 :: Only', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Topic :: Software Development :: Libraries :: Python Modules', + 'Topic :: Software Development :: Quality Assurance', +] +dependencies = ['platformdirs>=3.5.1', 'tomli>=2.0.1; python_version<"3.11"'] + +[project.scripts] +yapf = "yapf:run_main" +yapf-diff = "yapf_third_party.yapf_diff.yapf_diff:main" + +[project.urls] +# https://daniel.feldroy.com/posts/2023-08-pypi-project-urls-cheatsheet +Home = 'https://github.com/google/yapf' +Changelog = 'https://github.com/google/yapf/blob/main/CHANGELOG.md' +Docs = 'https://github.com/google/yapf/blob/main/README.md#yapf' +Issues = 'https://github.com/google/yapf/issues' + +[tool.distutils.bdist_wheel] +python_tag = "py3" + +[tool.setuptools] +include-package-data = true +package-dir = { yapf_third_party = 'third_party/yapf_third_party' } + +[tool.setuptools.dynamic] +version = { attr = "yapf._version.__version__" } + +[tool.setuptools.packages.find] +where = [".", 'third_party'] +include = ["yapf*", 'yapftests*'] + +[tool.setuptools.package-data] +yapf_third_party = [ + 'yapf_diff/LICENSE', + '_ylib2to3/Grammar.txt', + '_ylib2to3/PatternGrammar.txt', + '_ylib2to3/LICENSE', +] diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 2a9acf13d..000000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[bdist_wheel] -universal = 1 diff --git a/setup.py b/setup.py deleted file mode 100644 index 59042c009..000000000 --- a/setup.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python -# Copyright 2015-2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import codecs -import unittest -from setuptools import setup, Command -import sys - -import yapf - - -class RunTests(Command): - user_options = [] - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - loader = unittest.TestLoader() - tests = loader.discover('yapftests', pattern='*_test.py', top_level_dir='.') - runner = unittest.TextTestRunner() - results = runner.run(tests) - sys.exit(0 if results.wasSuccessful() else 1) - - -with codecs.open('README.rst', 'r', 'utf-8') as fd: - setup( - name='yapf', - version=yapf.__version__, - description='A formatter for Python code.', - long_description=fd.read(), - license='Apache License, Version 2.0', - author='Google Inc.', - maintainer='Bill Wendling', - maintainer_email='morbo@google.com', - packages=['yapf', 'yapf.yapflib', 'yapftests'], - classifiers=[ - 'Development Status :: 4 - Beta', - 'Environment :: Console', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: Apache Software License', - 'Operating System :: OS Independent', - 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Topic :: Software Development :: Libraries :: Python Modules', - 'Topic :: Software Development :: Quality Assurance', - ], - entry_points={'console_scripts': ['yapf = yapf:run_main'],}, - cmdclass={'test': RunTests,},) diff --git a/third_party/__init__.py b/third_party/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/third_party/yapf_third_party/__init__.py b/third_party/yapf_third_party/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/third_party/yapf_third_party/_ylib2to3/Grammar.txt b/third_party/yapf_third_party/_ylib2to3/Grammar.txt new file mode 100644 index 000000000..bd8a452a3 --- /dev/null +++ b/third_party/yapf_third_party/_ylib2to3/Grammar.txt @@ -0,0 +1,252 @@ +# Grammar for 2to3. This grammar supports Python 2.x and 3.x. + +# NOTE WELL: You should also follow all the steps listed at +# https://devguide.python.org/grammar/ + +# Start symbols for the grammar: +# file_input is a module or sequence of commands read from an input file; +# single_input is a single interactive statement; +# eval_input is the input for the eval() and input() functions. +# NB: compound_stmt in single_input is followed by extra NEWLINE! +file_input: (NEWLINE | stmt)* ENDMARKER +single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE +eval_input: testlist NEWLINE* ENDMARKER + +decorator: '@' namedexpr_test NEWLINE +decorators: decorator+ +decorated: decorators (classdef | funcdef | async_funcdef) +async_funcdef: ASYNC funcdef +funcdef: 'def' NAME parameters ['->' test] ':' suite +parameters: '(' [typedargslist] ')' + +# The following definition for typedarglist is equivalent to this set of rules: +# +# arguments = argument (',' argument)* +# argument = tfpdef ['=' test] +# kwargs = '**' tname [','] +# args = '*' [tname_star] +# kwonly_kwargs = (',' argument)* [',' [kwargs]] +# args_kwonly_kwargs = args kwonly_kwargs | kwargs +# poskeyword_args_kwonly_kwargs = arguments [',' [args_kwonly_kwargs]] +# typedargslist_no_posonly = poskeyword_args_kwonly_kwargs | args_kwonly_kwargs +# typedarglist = arguments ',' '/' [',' [typedargslist_no_posonly]])|(typedargslist_no_posonly)" +# +# It needs to be fully expanded to allow our LL(1) parser to work on it. + +typedargslist: tfpdef ['=' test] (',' tfpdef ['=' test])* ',' '/' [ + ',' [((tfpdef ['=' test] ',')* ('*' [tname_star] (',' tname ['=' test])* + [',' ['**' tname [',']]] | '**' tname [',']) + | tfpdef ['=' test] (',' tfpdef ['=' test])* [','])] + ] | ((tfpdef ['=' test] ',')* ('*' [tname_star] (',' tname ['=' test])* + [',' ['**' tname [',']]] | '**' tname [',']) + | tfpdef ['=' test] (',' tfpdef ['=' test])* [',']) + +tname: NAME [':' test] +tname_star: NAME [':' (test|star_expr)] +tfpdef: tname | '(' tfplist ')' +tfplist: tfpdef (',' tfpdef)* [','] + +# The following definition for varargslist is equivalent to this set of rules: +# +# arguments = argument (',' argument )* +# argument = vfpdef ['=' test] +# kwargs = '**' vname [','] +# args = '*' [vname] +# kwonly_kwargs = (',' argument )* [',' [kwargs]] +# args_kwonly_kwargs = args kwonly_kwargs | kwargs +# poskeyword_args_kwonly_kwargs = arguments [',' [args_kwonly_kwargs]] +# vararglist_no_posonly = poskeyword_args_kwonly_kwargs | args_kwonly_kwargs +# varargslist = arguments ',' '/' [','[(vararglist_no_posonly)]] | (vararglist_no_posonly) +# +# It needs to be fully expanded to allow our LL(1) parser to work on it. + +varargslist: vfpdef ['=' test ](',' vfpdef ['=' test])* ',' '/' [',' [ + ((vfpdef ['=' test] ',')* ('*' [vname] (',' vname ['=' test])* + [',' ['**' vname [',']]] | '**' vname [',']) + | vfpdef ['=' test] (',' vfpdef ['=' test])* [',']) + ]] | ((vfpdef ['=' test] ',')* + ('*' [vname] (',' vname ['=' test])* [',' ['**' vname [',']]]| '**' vname [',']) + | vfpdef ['=' test] (',' vfpdef ['=' test])* [',']) + +vname: NAME +vfpdef: vname | '(' vfplist ')' +vfplist: vfpdef (',' vfpdef)* [','] + +stmt: simple_stmt | compound_stmt +simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE +small_stmt: (expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt | + import_stmt | global_stmt | exec_stmt | assert_stmt) +expr_stmt: testlist_star_expr (annassign | augassign (yield_expr|testlist) | + ('=' (yield_expr|testlist_star_expr))*) +annassign: ':' test ['=' (yield_expr|testlist_star_expr)] +testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [','] +augassign: ('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' | + '<<=' | '>>=' | '**=' | '//=') +# For normal and annotated assignments, additional restrictions enforced by the interpreter +print_stmt: 'print' ( [ test (',' test)* [','] ] | + '>>' test [ (',' test)+ [','] ] ) +del_stmt: 'del' exprlist +pass_stmt: 'pass' +flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt +break_stmt: 'break' +continue_stmt: 'continue' +return_stmt: 'return' [testlist_star_expr] +yield_stmt: yield_expr +raise_stmt: 'raise' [test ['from' test | ',' test [',' test]]] +import_stmt: import_name | import_from +import_name: 'import' dotted_as_names +import_from: ('from' ('.'* dotted_name | '.'+) + 'import' ('*' | '(' import_as_names ')' | import_as_names)) +import_as_name: NAME ['as' NAME] +dotted_as_name: dotted_name ['as' NAME] +import_as_names: import_as_name (',' import_as_name)* [','] +dotted_as_names: dotted_as_name (',' dotted_as_name)* +dotted_name: NAME ('.' NAME)* +global_stmt: ('global' | 'nonlocal') NAME (',' NAME)* +exec_stmt: 'exec' expr ['in' test [',' test]] +assert_stmt: 'assert' test [',' test] + +compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt | match_stmt +async_stmt: ASYNC (funcdef | with_stmt | for_stmt) +if_stmt: 'if' namedexpr_test ':' suite ('elif' namedexpr_test ':' suite)* ['else' ':' suite] +while_stmt: 'while' namedexpr_test ':' suite ['else' ':' suite] +for_stmt: 'for' exprlist 'in' testlist_star_expr ':' suite ['else' ':' suite] +try_stmt: ('try' ':' suite + ((except_clause ':' suite)+ + ['else' ':' suite] + ['finally' ':' suite] | + 'finally' ':' suite)) +with_stmt: 'with' asexpr_test (',' asexpr_test)* ':' suite + +# NB compile.c makes sure that the default except clause is last +except_clause: 'except' ['*'] [test [(',' | 'as') test]] +suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT + +# Backward compatibility cruft to support: +# [ x for x in lambda: True, lambda: False if x() ] +# even while also allowing: +# lambda x: 5 if x else 2 +# (But not a mix of the two) +testlist_safe: old_test [(',' old_test)+ [',']] +old_test: or_test | old_lambdef +old_lambdef: 'lambda' [varargslist] ':' old_test + +namedexpr_test: asexpr_test [':=' asexpr_test] + +# This is actually not a real rule, though since the parser is very +# limited in terms of the strategy about match/case rules, we are inserting +# a virtual case ( as ) as a valid expression. Unless a better +# approach is thought, the only side effect of this seem to be just allowing +# more stuff to be parser (which would fail on the ast). +asexpr_test: test ['as' test] + +test: or_test ['if' or_test 'else' test] | lambdef +or_test: and_test ('or' and_test)* +and_test: not_test ('and' not_test)* +not_test: 'not' not_test | comparison +comparison: expr (comp_op expr)* +comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' +star_expr: '*' expr +expr: xor_expr ('|' xor_expr)* +xor_expr: and_expr ('^' and_expr)* +and_expr: shift_expr ('&' shift_expr)* +shift_expr: arith_expr (('<<'|'>>') arith_expr)* +arith_expr: term (('+'|'-') term)* +term: factor (('*'|'@'|'/'|'%'|'//') factor)* +factor: ('+'|'-'|'~') factor | power +power: [AWAIT] atom trailer* ['**' factor] +atom: ('(' [yield_expr|testlist_gexp] ')' | + '[' [listmaker] ']' | + '{' [dictsetmaker] '}' | + '`' testlist1 '`' | + NAME | NUMBER | STRING+ | '.' '.' '.') +listmaker: (namedexpr_test|star_expr) ( old_comp_for | (',' (namedexpr_test|star_expr))* [','] ) +testlist_gexp: (namedexpr_test|star_expr) ( old_comp_for | (',' (namedexpr_test|star_expr))* [','] ) +lambdef: 'lambda' [varargslist] ':' test +trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME +subscriptlist: (subscript|star_expr) (',' (subscript|star_expr))* [','] +subscript: test [':=' test] | [test] ':' [test] [sliceop] +sliceop: ':' [test] +exprlist: (expr|star_expr) (',' (expr|star_expr))* [','] +testlist: test (',' test)* [','] +dictsetmaker: ( ((test ':' asexpr_test | '**' expr) + (comp_for | (',' (test ':' asexpr_test | '**' expr))* [','])) | + ((test [':=' test] | star_expr) + (comp_for | (',' (test [':=' test] | star_expr))* [','])) ) + +classdef: 'class' NAME ['(' [arglist] ')'] ':' suite + +arglist: argument (',' argument)* [','] + +# "test '=' test" is really "keyword '=' test", but we have no such token. +# These need to be in a single rule to avoid grammar that is ambiguous +# to our LL(1) parser. Even though 'test' includes '*expr' in star_expr, +# we explicitly match '*' here, too, to give it proper precedence. +# Illegal combinations and orderings are blocked in ast.c: +# multiple (test comp_for) arguments are blocked; keyword unpackings +# that precede iterable unpackings are blocked; etc. +argument: ( test [comp_for] | + test ':=' test [comp_for] | + test 'as' test | + test '=' asexpr_test | + '**' test | + '*' test ) + +comp_iter: comp_for | comp_if +comp_for: [ASYNC] 'for' exprlist 'in' or_test [comp_iter] +comp_if: 'if' old_test [comp_iter] + +# As noted above, testlist_safe extends the syntax allowed in list +# comprehensions and generators. We can't use it indiscriminately in all +# derivations using a comp_for-like pattern because the testlist_safe derivation +# contains comma which clashes with trailing comma in arglist. +# +# This was an issue because the parser would not follow the correct derivation +# when parsing syntactically valid Python code. Since testlist_safe was created +# specifically to handle list comprehensions and generator expressions enclosed +# with parentheses, it's safe to only use it in those. That avoids the issue; we +# can parse code like set(x for x in [],). +# +# The syntax supported by this set of rules is not a valid Python 3 syntax, +# hence the prefix "old". +# +# See https://bugs.python.org/issue27494 +old_comp_iter: old_comp_for | old_comp_if +old_comp_for: [ASYNC] 'for' exprlist 'in' testlist_safe [old_comp_iter] +old_comp_if: 'if' old_test [old_comp_iter] + +testlist1: test (',' test)* + +# not used in grammar, but may appear in "node" passed from Parser to Compiler +encoding_decl: NAME + +yield_expr: 'yield' [yield_arg] +yield_arg: 'from' test | testlist_star_expr + + +# 3.10 match statement definition + +# PS: normally the grammar is much much more restricted, but +# at this moment for not trying to bother much with encoding the +# exact same DSL in a LL(1) parser, we will just accept an expression +# and let the ast.parse() step of the safe mode to reject invalid +# grammar. + +# The reason why it is more restricted is that, patterns are some +# sort of a DSL (more advanced than our LHS on assignments, but +# still in a very limited python subset). They are not really +# expressions, but who cares. If we can parse them, that is enough +# to reformat them. + +match_stmt: "match" subject_expr ':' NEWLINE INDENT case_block+ DEDENT + +# This is more permissive than the actual version. For example it +# accepts `match *something:`, even though single-item starred expressions +# are forbidden. +subject_expr: (namedexpr_test|star_expr) (',' (namedexpr_test|star_expr))* [','] + +# cases +case_block: "case" patterns [guard] ':' suite +guard: 'if' namedexpr_test +patterns: pattern (',' pattern)* [','] +pattern: (expr|star_expr) ['as' expr] diff --git a/third_party/yapf_third_party/_ylib2to3/LICENSE b/third_party/yapf_third_party/_ylib2to3/LICENSE new file mode 100644 index 000000000..ef8df0698 --- /dev/null +++ b/third_party/yapf_third_party/_ylib2to3/LICENSE @@ -0,0 +1,254 @@ +A. HISTORY OF THE SOFTWARE +========================== + +Python was created in the early 1990s by Guido van Rossum at Stichting +Mathematisch Centrum (CWI, see https://www.cwi.nl) in the Netherlands +as a successor of a language called ABC. Guido remains Python's +principal author, although it includes many contributions from others. + +In 1995, Guido continued his work on Python at the Corporation for +National Research Initiatives (CNRI, see https://www.cnri.reston.va.us) +in Reston, Virginia where he released several versions of the +software. + +In May 2000, Guido and the Python core development team moved to +BeOpen.com to form the BeOpen PythonLabs team. In October of the same +year, the PythonLabs team moved to Digital Creations, which became +Zope Corporation. In 2001, the Python Software Foundation (PSF, see +https://www.python.org/psf/) was formed, a non-profit organization +created specifically to own Python-related Intellectual Property. +Zope Corporation was a sponsoring member of the PSF. + +All Python releases are Open Source (see https://opensource.org for +the Open Source Definition). Historically, most, but not all, Python +releases have also been GPL-compatible; the table below summarizes +the various releases. + + Release Derived Year Owner GPL- + from compatible? (1) + + 0.9.0 thru 1.2 1991-1995 CWI yes + 1.3 thru 1.5.2 1.2 1995-1999 CNRI yes + 1.6 1.5.2 2000 CNRI no + 2.0 1.6 2000 BeOpen.com no + 1.6.1 1.6 2001 CNRI yes (2) + 2.1 2.0+1.6.1 2001 PSF no + 2.0.1 2.0+1.6.1 2001 PSF yes + 2.1.1 2.1+2.0.1 2001 PSF yes + 2.1.2 2.1.1 2002 PSF yes + 2.1.3 2.1.2 2002 PSF yes + 2.2 and above 2.1.1 2001-now PSF yes + +Footnotes: + +(1) GPL-compatible doesn't mean that we're distributing Python under + the GPL. All Python licenses, unlike the GPL, let you distribute + a modified version without making your changes open source. The + GPL-compatible licenses make it possible to combine Python with + other software that is released under the GPL; the others don't. + +(2) According to Richard Stallman, 1.6.1 is not GPL-compatible, + because its license has a choice of law clause. According to + CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1 + is "not incompatible" with the GPL. + +Thanks to the many outside volunteers who have worked under Guido's +direction to make these releases possible. + + +B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON +=============================================================== + +PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 +-------------------------------------------- + +1. This LICENSE AGREEMENT is between the Python Software Foundation +("PSF"), and the Individual or Organization ("Licensee") accessing and +otherwise using this software ("Python") in source or binary form and +its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, PSF hereby +grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, +analyze, test, perform and/or display publicly, prepare derivative works, +distribute, and otherwise use Python alone or in any derivative version, +provided, however, that PSF's License Agreement and PSF's notice of copyright, +i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, +2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018 Python Software Foundation; All +Rights Reserved" are retained in Python alone or in any derivative version +prepared by Licensee. + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python. + +4. PSF is making Python available to Licensee on an "AS IS" +basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. Nothing in this License Agreement shall be deemed to create any +relationship of agency, partnership, or joint venture between PSF and +Licensee. This License Agreement does not grant permission to use PSF +trademarks or trade name in a trademark sense to endorse or promote +products or services of Licensee, or any third party. + +8. By copying, installing or otherwise using Python, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0 +------------------------------------------- + +BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1 + +1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an +office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the +Individual or Organization ("Licensee") accessing and otherwise using +this software in source or binary form and its associated +documentation ("the Software"). + +2. Subject to the terms and conditions of this BeOpen Python License +Agreement, BeOpen hereby grants Licensee a non-exclusive, +royalty-free, world-wide license to reproduce, analyze, test, perform +and/or display publicly, prepare derivative works, distribute, and +otherwise use the Software alone or in any derivative version, +provided, however, that the BeOpen Python License is retained in the +Software, alone or in any derivative version prepared by Licensee. + +3. BeOpen is making the Software available to Licensee on an "AS IS" +basis. BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE +SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS +AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY +DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +5. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +6. This License Agreement shall be governed by and interpreted in all +respects by the law of the State of California, excluding conflict of +law provisions. Nothing in this License Agreement shall be deemed to +create any relationship of agency, partnership, or joint venture +between BeOpen and Licensee. This License Agreement does not grant +permission to use BeOpen trademarks or trade names in a trademark +sense to endorse or promote products or services of Licensee, or any +third party. As an exception, the "BeOpen Python" logos available at +http://www.pythonlabs.com/logos.html may be used according to the +permissions granted on that web page. + +7. By copying, installing or otherwise using the software, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1 +--------------------------------------- + +1. This LICENSE AGREEMENT is between the Corporation for National +Research Initiatives, having an office at 1895 Preston White Drive, +Reston, VA 20191 ("CNRI"), and the Individual or Organization +("Licensee") accessing and otherwise using Python 1.6.1 software in +source or binary form and its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, CNRI +hereby grants Licensee a nonexclusive, royalty-free, world-wide +license to reproduce, analyze, test, perform and/or display publicly, +prepare derivative works, distribute, and otherwise use Python 1.6.1 +alone or in any derivative version, provided, however, that CNRI's +License Agreement and CNRI's notice of copyright, i.e., "Copyright (c) +1995-2001 Corporation for National Research Initiatives; All Rights +Reserved" are retained in Python 1.6.1 alone or in any derivative +version prepared by Licensee. Alternately, in lieu of CNRI's License +Agreement, Licensee may substitute the following text (omitting the +quotes): "Python 1.6.1 is made available subject to the terms and +conditions in CNRI's License Agreement. This Agreement together with +Python 1.6.1 may be located on the Internet using the following +unique, persistent identifier (known as a handle): 1895.22/1013. This +Agreement may also be obtained from a proxy server on the Internet +using the following URL: http://hdl.handle.net/1895.22/1013". + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python 1.6.1 or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python 1.6.1. + +4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS" +basis. CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. This License Agreement shall be governed by the federal +intellectual property law of the United States, including without +limitation the federal copyright law, and, to the extent such +U.S. federal law does not apply, by the law of the Commonwealth of +Virginia, excluding Virginia's conflict of law provisions. +Notwithstanding the foregoing, with regard to derivative works based +on Python 1.6.1 that incorporate non-separable material that was +previously distributed under the GNU General Public License (GPL), the +law of the Commonwealth of Virginia shall govern this License +Agreement only as to issues arising under or with respect to +Paragraphs 4, 5, and 7 of this License Agreement. Nothing in this +License Agreement shall be deemed to create any relationship of +agency, partnership, or joint venture between CNRI and Licensee. This +License Agreement does not grant permission to use CNRI trademarks or +trade name in a trademark sense to endorse or promote products or +services of Licensee, or any third party. + +8. By clicking on the "ACCEPT" button where indicated, or by copying, +installing or otherwise using Python 1.6.1, Licensee agrees to be +bound by the terms and conditions of this License Agreement. + + ACCEPT + + +CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 +-------------------------------------------------- + +Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam, +The Netherlands. All rights reserved. + +Permission to use, copy, modify, and distribute this software and its +documentation for any purpose and without fee is hereby granted, +provided that the above copyright notice appear in all copies and that +both that copyright notice and this permission notice appear in +supporting documentation, and that the name of Stichting Mathematisch +Centrum or CWI not be used in advertising or publicity pertaining to +distribution of the software without specific, written prior +permission. + +STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO +THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE +FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT +OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/third_party/yapf_third_party/_ylib2to3/PatternGrammar.txt b/third_party/yapf_third_party/_ylib2to3/PatternGrammar.txt new file mode 100644 index 000000000..36bf81482 --- /dev/null +++ b/third_party/yapf_third_party/_ylib2to3/PatternGrammar.txt @@ -0,0 +1,28 @@ +# Copyright 2006 Google, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. + +# A grammar to describe tree matching patterns. +# Not shown here: +# - 'TOKEN' stands for any token (leaf node) +# - 'any' stands for any node (leaf or interior) +# With 'any' we can still specify the sub-structure. + +# The start symbol is 'Matcher'. + +Matcher: Alternatives ENDMARKER + +Alternatives: Alternative ('|' Alternative)* + +Alternative: (Unit | NegatedUnit)+ + +Unit: [NAME '='] ( STRING [Repeater] + | NAME [Details] [Repeater] + | '(' Alternatives ')' [Repeater] + | '[' Alternatives ']' + ) + +NegatedUnit: 'not' (STRING | NAME [Details] | '(' Alternatives ')') + +Repeater: '*' | '+' | '{' NUMBER [',' NUMBER] '}' + +Details: '<' Alternatives '>' diff --git a/third_party/yapf_third_party/_ylib2to3/README.rst b/third_party/yapf_third_party/_ylib2to3/README.rst new file mode 100644 index 000000000..21d69b89a --- /dev/null +++ b/third_party/yapf_third_party/_ylib2to3/README.rst @@ -0,0 +1,9 @@ +A fork of python's lib2to3 with select features backported from black's blib2to3. + +Reasons for forking: + +- black's fork of lib2to3 already considers newer features like Structured Pattern matching +- lib2to3 itself is deprecated and no longer getting support + +Maintenance moving forward: +- Most changes moving forward should only have to be done to the grammar files in this project. diff --git a/third_party/yapf_third_party/_ylib2to3/__init__.py b/third_party/yapf_third_party/_ylib2to3/__init__.py new file mode 100644 index 000000000..1de94366c --- /dev/null +++ b/third_party/yapf_third_party/_ylib2to3/__init__.py @@ -0,0 +1 @@ +"""fork of python's lib2to3 with some backports from black's blib2to3""" diff --git a/third_party/yapf_third_party/_ylib2to3/fixer_base.py b/third_party/yapf_third_party/_ylib2to3/fixer_base.py new file mode 100644 index 000000000..92fd0f699 --- /dev/null +++ b/third_party/yapf_third_party/_ylib2to3/fixer_base.py @@ -0,0 +1,187 @@ +# Copyright 2006 Google, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. +"""Base class for fixers (optional, but recommended).""" + +# Python imports +import itertools + +from . import pygram +from .fixer_util import does_tree_import +# Local imports +from .patcomp import PatternCompiler + + +class BaseFix(object): + """Optional base class for fixers. + + The subclass name must be FixFooBar where FooBar is the result of + removing underscores and capitalizing the words of the fix name. + For example, the class name for a fixer named 'has_key' should be + FixHasKey. + """ + + PATTERN = None # Most subclasses should override with a string literal + pattern = None # Compiled pattern, set by compile_pattern() + pattern_tree = None # Tree representation of the pattern + options = None # Options object passed to initializer + filename = None # The filename (set by set_filename) + numbers = itertools.count(1) # For new_name() + used_names = set() # A set of all used NAMEs + order = 'post' # Does the fixer prefer pre- or post-order traversal + explicit = False # Is this ignored by refactor.py -f all? + run_order = 5 # Fixers will be sorted by run order before execution + # Lower numbers will be run first. + _accept_type = None # [Advanced and not public] This tells RefactoringTool + # which node type to accept when there's not a pattern. + + keep_line_order = False # For the bottom matcher: match with the + # original line order + BM_compatible = False # Compatibility with the bottom matching + # module; every fixer should set this + # manually + + # Shortcut for access to Python grammar symbols + syms = pygram.python_symbols + + def __init__(self, options, log): + """Initializer. Subclass may override. + + Args: + options: a dict containing the options passed to RefactoringTool + that could be used to customize the fixer through the command line. + log: a list to append warnings and other messages to. + """ + self.options = options + self.log = log + self.compile_pattern() + + def compile_pattern(self): + """Compiles self.PATTERN into self.pattern. + + Subclass may override if it doesn't want to use + self.{pattern,PATTERN} in .match(). + """ + if self.PATTERN is not None: + PC = PatternCompiler() + self.pattern, self.pattern_tree = PC.compile_pattern( + self.PATTERN, with_tree=True) + + def set_filename(self, filename): + """Set the filename. + + The main refactoring tool should call this. + """ + self.filename = filename + + def match(self, node): + """Returns match for a given parse tree node. + + Should return a true or false object (not necessarily a bool). + It may return a non-empty dict of matching sub-nodes as + returned by a matching pattern. + + Subclass may override. + """ + results = {'node': node} + return self.pattern.match(node, results) and results + + def transform(self, node, results): + """Returns the transformation for a given parse tree node. + + Args: + node: the root of the parse tree that matched the fixer. + results: a dict mapping symbolic names to part of the match. + + Returns: + None, or a node that is a modified copy of the + argument node. The node argument may also be modified in-place to + effect the same change. + + Subclass *must* override. + """ + raise NotImplementedError() + + def new_name(self, template='xxx_todo_changeme'): + """Return a string suitable for use as an identifier + + The new name is guaranteed not to conflict with other identifiers. + """ + name = template + while name in self.used_names: + name = template + str(next(self.numbers)) + self.used_names.add(name) + return name + + def log_message(self, message): + if self.first_log: + self.first_log = False + self.log.append('### In file %s ###' % self.filename) + self.log.append(message) + + def cannot_convert(self, node, reason=None): + """Warn the user that a given chunk of code is not valid Python 3, + but that it cannot be converted automatically. + + First argument is the top-level node for the code in question. + Optional second argument is why it can't be converted. + """ + lineno = node.get_lineno() + for_output = node.clone() + for_output.prefix = '' + msg = 'Line %d: could not convert: %s' + self.log_message(msg % (lineno, for_output)) + if reason: + self.log_message(reason) + + def warning(self, node, reason): + """Used for warning the user about possible uncertainty in the translation. + + First argument is the top-level node for the code in question. + Optional second argument is why it can't be converted. + """ + lineno = node.get_lineno() + self.log_message('Line %d: %s' % (lineno, reason)) + + def start_tree(self, tree, filename): + """Some fixers need to maintain tree-wide state. + + This method is called once, at the start of tree fix-up. + + tree - the root node of the tree to be processed. + filename - the name of the file the tree came from. + """ + self.used_names = tree.used_names + self.set_filename(filename) + self.numbers = itertools.count(1) + self.first_log = True + + def finish_tree(self, tree, filename): + """Some fixers need to maintain tree-wide state. + + This method is called once, at the conclusion of tree fix-up. + + tree - the root node of the tree to be processed. + filename - the name of the file the tree came from. + """ + pass + + +class ConditionalFix(BaseFix): + """ Base class for fixers which not execute if an import is found. """ + + # This is the name of the import which, if found, will cause the test to be + # skipped. + skip_on = None + + def start_tree(self, *args): + super(ConditionalFix, self).start_tree(*args) + self._should_skip = None + + def should_skip(self, node): + if self._should_skip is not None: + return self._should_skip + pkg = self.skip_on.split('.') + name = pkg[-1] + pkg = '.'.join(pkg[:-1]) + self._should_skip = does_tree_import(pkg, name, node) + return self._should_skip diff --git a/third_party/yapf_third_party/_ylib2to3/fixer_util.py b/third_party/yapf_third_party/_ylib2to3/fixer_util.py new file mode 100644 index 000000000..373b2be17 --- /dev/null +++ b/third_party/yapf_third_party/_ylib2to3/fixer_util.py @@ -0,0 +1,493 @@ +"""Utility functions, node construction macros, etc.""" +# Author: Collin Winter + +from . import patcomp +# Local imports +from .pgen2 import token +from .pygram import python_symbols as syms +from .pytree import Leaf +from .pytree import Node + +########################################################### +# Common node-construction "macros" +########################################################### + + +def KeywordArg(keyword, value): + return Node(syms.argument, [keyword, Leaf(token.EQUAL, '='), value]) + + +def LParen(): + return Leaf(token.LPAR, '(') + + +def RParen(): + return Leaf(token.RPAR, ')') + + +def Assign(target, source): + """Build an assignment statement""" + if not isinstance(target, list): + target = [target] + if not isinstance(source, list): + source.prefix = ' ' + source = [source] + + return Node(syms.atom, target + [Leaf(token.EQUAL, '=', prefix=' ')] + source) + + +def Name(name, prefix=None): + """Return a NAME leaf""" + return Leaf(token.NAME, name, prefix=prefix) + + +def Attr(obj, attr): + """A node tuple for obj.attr""" + return [obj, Node(syms.trailer, [Dot(), attr])] + + +def Comma(): + """A comma leaf""" + return Leaf(token.COMMA, ',') + + +def Dot(): + """A period (.) leaf""" + return Leaf(token.DOT, '.') + + +def ArgList(args, lparen=LParen(), rparen=RParen()): + """A parenthesised argument list, used by Call()""" + node = Node(syms.trailer, [lparen.clone(), rparen.clone()]) + if args: + node.insert_child(1, Node(syms.arglist, args)) + return node + + +def Call(func_name, args=None, prefix=None): + """A function call""" + node = Node(syms.power, [func_name, ArgList(args)]) + if prefix is not None: + node.prefix = prefix + return node + + +def Newline(): + """A newline literal""" + return Leaf(token.NEWLINE, '\n') + + +def BlankLine(): + """A blank line""" + return Leaf(token.NEWLINE, '') + + +def Number(n, prefix=None): + return Leaf(token.NUMBER, n, prefix=prefix) + + +def Subscript(index_node): + """A numeric or string subscript""" + return Node(syms.trailer, + [Leaf(token.LBRACE, '['), index_node, + Leaf(token.RBRACE, ']')]) + + +def String(string, prefix=None): + """A string leaf""" + return Leaf(token.STRING, string, prefix=prefix) + + +def ListComp(xp, fp, it, test=None): + """A list comprehension of the form [xp for fp in it if test]. + + If test is None, the "if test" part is omitted. + """ + xp.prefix = '' + fp.prefix = ' ' + it.prefix = ' ' + for_leaf = Leaf(token.NAME, 'for') + for_leaf.prefix = ' ' + in_leaf = Leaf(token.NAME, 'in') + in_leaf.prefix = ' ' + inner_args = [for_leaf, fp, in_leaf, it] + if test: + test.prefix = ' ' + if_leaf = Leaf(token.NAME, 'if') + if_leaf.prefix = ' ' + inner_args.append(Node(syms.comp_if, [if_leaf, test])) + inner = Node(syms.listmaker, [xp, Node(syms.comp_for, inner_args)]) + return Node(syms.atom, + [Leaf(token.LBRACE, '['), inner, + Leaf(token.RBRACE, ']')]) + + +def FromImport(package_name, name_leafs): + """ Return an import statement in the form: + + from package import name_leafs + """ + # XXX: May not handle dotted imports properly (eg, package_name='foo.bar') + # #assert package_name == '.' or '.' not in package_name, "FromImport has "\ + # "not been tested with dotted package names -- use at your own "\ + # "peril!" + + for leaf in name_leafs: + # Pull the leaves out of their old tree + leaf.remove() + + children = [ + Leaf(token.NAME, 'from'), + Leaf(token.NAME, package_name, prefix=' '), + Leaf(token.NAME, 'import', prefix=' '), + Node(syms.import_as_names, name_leafs) + ] + imp = Node(syms.import_from, children) + return imp + + +def ImportAndCall(node, results, names): + """Returns an import statement and calls a method of the module: + + import module + module.name() + """ + obj = results['obj'].clone() + if obj.type == syms.arglist: + newarglist = obj.clone() + else: + newarglist = Node(syms.arglist, [obj.clone()]) + after = results['after'] + if after: + after = [n.clone() for n in after] + new = Node( + syms.power, + Attr(Name(names[0]), Name(names[1])) + [ + Node(syms.trailer, + [results['lpar'].clone(), newarglist, results['rpar'].clone()]) + ] + after) + new.prefix = node.prefix + return new + + +########################################################### +# Determine whether a node represents a given literal +########################################################### + + +def is_tuple(node): + """Does the node represent a tuple literal?""" + if isinstance(node, Node) and node.children == [LParen(), RParen()]: + return True + return (isinstance(node, Node) and len(node.children) == 3 and + isinstance(node.children[0], Leaf) and + isinstance(node.children[1], Node) and + isinstance(node.children[2], Leaf) and + node.children[0].value == '(' and node.children[2].value == ')') + + +def is_list(node): + """Does the node represent a list literal?""" + return (isinstance(node, Node) and len(node.children) > 1 and + isinstance(node.children[0], Leaf) and + isinstance(node.children[-1], Leaf) and + node.children[0].value == '[' and node.children[-1].value == ']') + + +########################################################### +# Misc +########################################################### + + +def parenthesize(node): + return Node(syms.atom, [LParen(), node, RParen()]) + + +consuming_calls = { + 'sorted', 'list', 'set', 'any', 'all', 'tuple', 'sum', 'min', 'max', + 'enumerate' +} + + +def attr_chain(obj, attr): + """Follow an attribute chain. + + If you have a chain of objects where a.foo -> b, b.foo-> c, etc, use this to + iterate over all objects in the chain. Iteration is terminated by getattr(x, + attr) is None. + + Args: + obj: the starting object + attr: the name of the chaining attribute + + Yields: + Each successive object in the chain. + """ + next = getattr(obj, attr) + while next: + yield next + next = getattr(next, attr) + + +p0 = """for_stmt< 'for' any 'in' node=any ':' any* > + | comp_for< 'for' any 'in' node=any any* > + """ +p1 = """ +power< + ( 'iter' | 'list' | 'tuple' | 'sorted' | 'set' | 'sum' | + 'any' | 'all' | 'enumerate' | (any* trailer< '.' 'join' >) ) + trailer< '(' node=any ')' > + any* +> +""" +p2 = """ +power< + ( 'sorted' | 'enumerate' ) + trailer< '(' arglist ')' > + any* +> +""" +pats_built = False + + +def in_special_context(node): + """ Returns true if node is in an environment where all that is required + of it is being iterable (ie, it doesn't matter if it returns a list + or an iterator). + See test_map_nochange in test_fixers.py for some examples and tests. + """ + global p0, p1, p2, pats_built + if not pats_built: + p0 = patcomp.compile_pattern(p0) + p1 = patcomp.compile_pattern(p1) + p2 = patcomp.compile_pattern(p2) + pats_built = True + patterns = [p0, p1, p2] + for pattern, parent in zip(patterns, attr_chain(node, 'parent')): + results = {} + if pattern.match(parent, results) and results['node'] is node: + return True + return False + + +def is_probably_builtin(node): + """Check that something isn't an attribute or function name etc.""" + prev = node.prev_sibling + if prev is not None and prev.type == token.DOT: + # Attribute lookup. + return False + parent = node.parent + if parent.type in (syms.funcdef, syms.classdef): + return False + if parent.type == syms.expr_stmt and parent.children[0] is node: + # Assignment. + return False + if parent.type == syms.parameters or (parent.type == syms.typedargslist and ( + (prev is not None and prev.type == token.COMMA) or + parent.children[0] is node)): + # The name of an argument. + return False + return True + + +def find_indentation(node): + """Find the indentation of *node*.""" + while node is not None: + if node.type == syms.suite and len(node.children) > 2: + indent = node.children[1] + if indent.type == token.INDENT: + return indent.value + node = node.parent + return '' + + +########################################################### +# The following functions are to find bindings in a suite +########################################################### + + +def make_suite(node): + if node.type == syms.suite: + return node + node = node.clone() + parent, node.parent = node.parent, None + suite = Node(syms.suite, [node]) + suite.parent = parent + return suite + + +def find_root(node): + """Find the top level namespace.""" + # Scamper up to the top level namespace + while node.type != syms.file_input: + node = node.parent + if not node: + raise ValueError('root found before file_input node was found.') + return node + + +def does_tree_import(package, name, node): + """ Returns true if name is imported from package at the + top level of the tree which node belongs to. + To cover the case of an import like 'import foo', use + None for the package and 'foo' for the name. + """ + binding = find_binding(name, find_root(node), package) + return bool(binding) + + +def is_import(node): + """Returns true if the node is an import statement.""" + return node.type in (syms.import_name, syms.import_from) + + +def touch_import(package, name, node): + """ Works like `does_tree_import` but adds an import statement + if it was not imported. """ + + def is_import_stmt(node): + return (node.type == syms.simple_stmt and node.children and + is_import(node.children[0])) + + root = find_root(node) + + if does_tree_import(package, name, root): + return + + # figure out where to insert the new import. First try to find + # the first import and then skip to the last one. + insert_pos = offset = 0 + for idx, node in enumerate(root.children): + if not is_import_stmt(node): + continue + for offset, node2 in enumerate(root.children[idx:]): + if not is_import_stmt(node2): + break + insert_pos = idx + offset + break + + # if there are no imports where we can insert, find the docstring. + # if that also fails, we stick to the beginning of the file + if insert_pos == 0: + for idx, node in enumerate(root.children): + if (node.type == syms.simple_stmt and node.children and + node.children[0].type == token.STRING): + insert_pos = idx + 1 + break + + if package is None: + import_ = Node( + syms.import_name, + [Leaf(token.NAME, 'import'), + Leaf(token.NAME, name, prefix=' ')]) + else: + import_ = FromImport(package, [Leaf(token.NAME, name, prefix=' ')]) + + children = [import_, Newline()] + root.insert_child(insert_pos, Node(syms.simple_stmt, children)) + + +_def_syms = {syms.classdef, syms.funcdef} + + +def find_binding(name, node, package=None): + """ Returns the node which binds variable name, otherwise None. + If optional argument package is supplied, only imports will + be returned. + See test cases for examples. + """ + for child in node.children: + ret = None + if child.type == syms.for_stmt: + if _find(name, child.children[1]): + return child + n = find_binding(name, make_suite(child.children[-1]), package) + if n: + ret = n + elif child.type in (syms.if_stmt, syms.while_stmt): + n = find_binding(name, make_suite(child.children[-1]), package) + if n: + ret = n + elif child.type == syms.try_stmt: + n = find_binding(name, make_suite(child.children[2]), package) + if n: + ret = n + else: + for i, kid in enumerate(child.children[3:]): + if kid.type == token.COLON and kid.value == ':': + # i+3 is the colon, i+4 is the suite + n = find_binding(name, make_suite(child.children[i + 4]), package) + if n: + ret = n + elif child.type in _def_syms and child.children[1].value == name: + ret = child + elif _is_import_binding(child, name, package): + ret = child + elif child.type == syms.simple_stmt: + ret = find_binding(name, child, package) + elif child.type == syms.expr_stmt: + if _find(name, child.children[0]): + ret = child + + if ret: + if not package: + return ret + if is_import(ret): + return ret + return None + + +_block_syms = {syms.funcdef, syms.classdef, syms.trailer} + + +def _find(name, node): + nodes = [node] + while nodes: + node = nodes.pop() + if node.type > 256 and node.type not in _block_syms: + nodes.extend(node.children) + elif node.type == token.NAME and node.value == name: + return node + return None + + +def _is_import_binding(node, name, package=None): + """ Will return node if node will import name, or node + will import * from package. None is returned otherwise. + See test cases for examples. + """ + if node.type == syms.import_name and not package: + imp = node.children[1] + if imp.type == syms.dotted_as_names: + for child in imp.children: + if child.type == syms.dotted_as_name: + if child.children[2].value == name: + return node + elif child.type == token.NAME and child.value == name: + return node + elif imp.type == syms.dotted_as_name: + last = imp.children[-1] + if last.type == token.NAME and last.value == name: + return node + elif imp.type == token.NAME and imp.value == name: + return node + elif node.type == syms.import_from: + # str(...) is used to make life easier here, because + # from a.b import parses to ['import', ['a', '.', 'b'], ...] + if package and str(node.children[1]).strip() != package: + return None + n = node.children[3] + if package and _find('as', n): + # See test_from_import_as for explanation + return None + elif n.type == syms.import_as_names and _find(name, n): + return node + elif n.type == syms.import_as_name: + child = n.children[2] + if child.type == token.NAME and child.value == name: + return node + elif n.type == token.NAME and n.value == name: + return node + elif package and n.type == token.STAR: + return node + return None diff --git a/third_party/yapf_third_party/_ylib2to3/patcomp.py b/third_party/yapf_third_party/_ylib2to3/patcomp.py new file mode 100644 index 000000000..20b7893d3 --- /dev/null +++ b/third_party/yapf_third_party/_ylib2to3/patcomp.py @@ -0,0 +1,209 @@ +# Copyright 2006 Google, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. +"""Pattern compiler. + +The grammar is taken from PatternGrammar.txt. + +The compiler compiles a pattern to a pytree.*Pattern instance. +""" + +__author__ = 'Guido van Rossum ' + +# Python imports +import io + +# Really local imports +from . import pygram +from . import pytree +# Fairly local imports +from .pgen2 import driver +from .pgen2 import grammar +from .pgen2 import literals +from .pgen2 import parse +from .pgen2 import token +from .pgen2 import tokenize + + +class PatternSyntaxError(Exception): + pass + + +def tokenize_wrapper(input): + """Tokenizes a string suppressing significant whitespace.""" + skip = {token.NEWLINE, token.INDENT, token.DEDENT} + tokens = tokenize.generate_tokens(io.StringIO(input).readline) + for quintuple in tokens: + type, value, start, end, line_text = quintuple + if type not in skip: + yield quintuple + + +class PatternCompiler(object): + + def __init__(self, grammar_file=None): + """Initializer. + + Takes an optional alternative filename for the pattern grammar. + """ + if grammar_file is None: + self.grammar = pygram.pattern_grammar + self.syms = pygram.pattern_symbols + else: + self.grammar = driver.load_grammar(grammar_file) + self.syms = pygram.Symbols(self.grammar) + self.pygrammar = pygram.python_grammar + self.pysyms = pygram.python_symbols + self.driver = driver.Driver(self.grammar, convert=pattern_convert) + + def compile_pattern(self, input, debug=False, with_tree=False): + """Compiles a pattern string to a nested pytree.*Pattern object.""" + tokens = tokenize_wrapper(input) + try: + root = self.driver.parse_tokens(tokens, debug=debug) + except parse.ParseError as e: + raise PatternSyntaxError(str(e)) from None + if with_tree: + return self.compile_node(root), root + else: + return self.compile_node(root) + + def compile_node(self, node): + """Compiles a node, recursively. + + This is one big switch on the node type. + """ + # XXX Optimize certain Wildcard-containing-Wildcard patterns + # that can be merged + if node.type == self.syms.Matcher: + node = node.children[0] # Avoid unneeded recursion + + if node.type == self.syms.Alternatives: + # Skip the odd children since they are just '|' tokens + alts = [self.compile_node(ch) for ch in node.children[::2]] + if len(alts) == 1: + return alts[0] + p = pytree.WildcardPattern([[a] for a in alts], min=1, max=1) + return p.optimize() + + if node.type == self.syms.Alternative: + units = [self.compile_node(ch) for ch in node.children] + if len(units) == 1: + return units[0] + p = pytree.WildcardPattern([units], min=1, max=1) + return p.optimize() + + if node.type == self.syms.NegatedUnit: + pattern = self.compile_basic(node.children[1:]) + p = pytree.NegatedPattern(pattern) + return p.optimize() + + assert node.type == self.syms.Unit + + name = None + nodes = node.children + if len(nodes) >= 3 and nodes[1].type == token.EQUAL: + name = nodes[0].value + nodes = nodes[2:] + repeat = None + if len(nodes) >= 2 and nodes[-1].type == self.syms.Repeater: + repeat = nodes[-1] + nodes = nodes[:-1] + + # Now we've reduced it to: STRING | NAME [Details] | (...) | [...] + pattern = self.compile_basic(nodes, repeat) + + if repeat is not None: + assert repeat.type == self.syms.Repeater + children = repeat.children + child = children[0] + if child.type == token.STAR: + min = 0 + max = pytree.HUGE + elif child.type == token.PLUS: + min = 1 + max = pytree.HUGE + elif child.type == token.LBRACE: + assert children[-1].type == token.RBRACE + assert len(children) in (3, 5) + min = max = self.get_int(children[1]) + if len(children) == 5: + max = self.get_int(children[3]) + else: + assert False + if min != 1 or max != 1: + pattern = pattern.optimize() + pattern = pytree.WildcardPattern([[pattern]], min=min, max=max) + + if name is not None: + pattern.name = name + return pattern.optimize() + + def compile_basic(self, nodes, repeat=None): + # Compile STRING | NAME [Details] | (...) | [...] + assert len(nodes) >= 1 + node = nodes[0] + if node.type == token.STRING: + value = str(literals.evalString(node.value)) + return pytree.LeafPattern(_type_of_literal(value), value) + elif node.type == token.NAME: + value = node.value + if value.isupper(): + if value not in TOKEN_MAP: + raise PatternSyntaxError('Invalid token: %r' % value) + if nodes[1:]: + raise PatternSyntaxError("Can't have details for token") + return pytree.LeafPattern(TOKEN_MAP[value]) + else: + if value == 'any': + type = None + elif not value.startswith('_'): + type = getattr(self.pysyms, value, None) + if type is None: + raise PatternSyntaxError('Invalid symbol: %r' % value) + if nodes[1:]: # Details present + content = [self.compile_node(nodes[1].children[1])] + else: + content = None + return pytree.NodePattern(type, content) + elif node.value == '(': + return self.compile_node(nodes[1]) + elif node.value == '[': + assert repeat is None + subpattern = self.compile_node(nodes[1]) + return pytree.WildcardPattern([[subpattern]], min=0, max=1) + assert False, node + + def get_int(self, node): + assert node.type == token.NUMBER + return int(node.value) + + +# Map named tokens to the type value for a LeafPattern +TOKEN_MAP = { + 'NAME': token.NAME, + 'STRING': token.STRING, + 'NUMBER': token.NUMBER, + 'TOKEN': None +} + + +def _type_of_literal(value): + if value[0].isalpha(): + return token.NAME + elif value in grammar.opmap: + return grammar.opmap[value] + else: + return None + + +def pattern_convert(grammar, raw_node_info): + """Converts raw node information to a Node or Leaf instance.""" + type, value, context, children = raw_node_info + if children or type in grammar.number2symbol: + return pytree.Node(type, children, context=context) + else: + return pytree.Leaf(type, value, context=context) + + +def compile_pattern(pattern): + return PatternCompiler().compile_pattern(pattern) diff --git a/third_party/yapf_third_party/_ylib2to3/pgen2/__init__.py b/third_party/yapf_third_party/_ylib2to3/pgen2/__init__.py new file mode 100644 index 000000000..7c8380a16 --- /dev/null +++ b/third_party/yapf_third_party/_ylib2to3/pgen2/__init__.py @@ -0,0 +1,3 @@ +# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. +"""The pgen2 package.""" diff --git a/third_party/yapf_third_party/_ylib2to3/pgen2/conv.py b/third_party/yapf_third_party/_ylib2to3/pgen2/conv.py new file mode 100644 index 000000000..a446771b8 --- /dev/null +++ b/third_party/yapf_third_party/_ylib2to3/pgen2/conv.py @@ -0,0 +1,254 @@ +# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. +"""Convert graminit.[ch] spit out by pgen to Python code. + +Pgen is the Python parser generator. It is useful to quickly create a +parser from a grammar file in Python's grammar notation. But I don't +want my parsers to be written in C (yet), so I'm translating the +parsing tables to Python data structures and writing a Python parse +engine. + +Note that the token numbers are constants determined by the standard +Python tokenizer. The standard token module defines these numbers and +their names (the names are not used much). The token numbers are +hardcoded into the Python tokenizer and into pgen. A Python +implementation of the Python tokenizer is also available, in the +standard tokenize module. + +On the other hand, symbol numbers (representing the grammar's +non-terminals) are assigned by pgen based on the actual grammar +input. + +Note: this module is pretty much obsolete; the pgen module generates +equivalent grammar tables directly from the Grammar.txt input file +without having to invoke the Python pgen C program. + +""" + +# Python imports +import re + +# Local imports +from pgen2 import grammar +from pgen2 import token + + +class Converter(grammar.Grammar): + """Grammar subclass that reads classic pgen output files. + + The run() method reads the tables as produced by the pgen parser + generator, typically contained in two C files, graminit.h and + graminit.c. The other methods are for internal use only. + + See the base class for more documentation. + + """ + + def run(self, graminit_h, graminit_c): + """Load the grammar tables from the text files written by pgen.""" + self.parse_graminit_h(graminit_h) + self.parse_graminit_c(graminit_c) + self.finish_off() + + def parse_graminit_h(self, filename): + """Parse the .h file written by pgen. (Internal) + + This file is a sequence of #define statements defining the + nonterminals of the grammar as numbers. We build two tables + mapping the numbers to names and back. + + """ + try: + f = open(filename) + except OSError as err: + print("Can't open %s: %s" % (filename, err)) + return False + self.symbol2number = {} + self.number2symbol = {} + lineno = 0 + for line in f: + lineno += 1 + mo = re.match(r'^#define\s+(\w+)\s+(\d+)$', line) + if not mo and line.strip(): + print("%s(%s): can't parse %s" % (filename, lineno, line.strip())) + else: + symbol, number = mo.groups() + number = int(number) + assert symbol not in self.symbol2number + assert number not in self.number2symbol + self.symbol2number[symbol] = number + self.number2symbol[number] = symbol + return True + + def parse_graminit_c(self, filename): + """Parse the .c file written by pgen. (Internal) + + The file looks as follows. The first two lines are always this: + + #include "pgenheaders.h" + #include "grammar.h" + + After that come four blocks: + + 1) one or more state definitions + 2) a table defining dfas + 3) a table defining labels + 4) a struct defining the grammar + + A state definition has the following form: + - one or more arc arrays, each of the form: + static arc arcs__[] = { + {, }, + ... + }; + - followed by a state array, of the form: + static state states_[] = { + {, arcs__}, + ... + }; + + """ + try: + f = open(filename) + except OSError as err: + print("Can't open %s: %s" % (filename, err)) + return False + # The code below essentially uses f's iterator-ness! + lineno = 0 + + # Expect the two #include lines + lineno, line = lineno + 1, next(f) + assert line == '#include "pgenheaders.h"\n', (lineno, line) + lineno, line = lineno + 1, next(f) + assert line == '#include "grammar.h"\n', (lineno, line) + + # Parse the state definitions + lineno, line = lineno + 1, next(f) + allarcs = {} + states = [] + while line.startswith('static arc '): + while line.startswith('static arc '): + mo = re.match(r'static arc arcs_(\d+)_(\d+)\[(\d+)\] = {$', line) + assert mo, (lineno, line) + n, m, k = list(map(int, mo.groups())) + arcs = [] + for _ in range(k): + lineno, line = lineno + 1, next(f) + mo = re.match(r'\s+{(\d+), (\d+)},$', line) + assert mo, (lineno, line) + i, j = list(map(int, mo.groups())) + arcs.append((i, j)) + lineno, line = lineno + 1, next(f) + assert line == '};\n', (lineno, line) + allarcs[(n, m)] = arcs + lineno, line = lineno + 1, next(f) + mo = re.match(r'static state states_(\d+)\[(\d+)\] = {$', line) + assert mo, (lineno, line) + s, t = list(map(int, mo.groups())) + assert s == len(states), (lineno, line) + state = [] + for _ in range(t): + lineno, line = lineno + 1, next(f) + mo = re.match(r'\s+{(\d+), arcs_(\d+)_(\d+)},$', line) + assert mo, (lineno, line) + k, n, m = list(map(int, mo.groups())) + arcs = allarcs[n, m] + assert k == len(arcs), (lineno, line) + state.append(arcs) + states.append(state) + lineno, line = lineno + 1, next(f) + assert line == '};\n', (lineno, line) + lineno, line = lineno + 1, next(f) + self.states = states + + # Parse the dfas + dfas = {} + mo = re.match(r'static dfa dfas\[(\d+)\] = {$', line) + assert mo, (lineno, line) + ndfas = int(mo.group(1)) + for i in range(ndfas): + lineno, line = lineno + 1, next(f) + mo = re.match(r'\s+{(\d+), "(\w+)", (\d+), (\d+), states_(\d+),$', line) + assert mo, (lineno, line) + symbol = mo.group(2) + number, x, y, z = list(map(int, mo.group(1, 3, 4, 5))) + assert self.symbol2number[symbol] == number, (lineno, line) + assert self.number2symbol[number] == symbol, (lineno, line) + assert x == 0, (lineno, line) + state = states[z] + assert y == len(state), (lineno, line) + lineno, line = lineno + 1, next(f) + mo = re.match(r'\s+("(?:\\\d\d\d)*")},$', line) + assert mo, (lineno, line) + first = {} + rawbitset = eval(mo.group(1)) + for i, c in enumerate(rawbitset): + byte = ord(c) + for j in range(8): + if byte & (1 << j): + first[i * 8 + j] = 1 + dfas[number] = (state, first) + lineno, line = lineno + 1, next(f) + assert line == '};\n', (lineno, line) + self.dfas = dfas + + # Parse the labels + labels = [] + lineno, line = lineno + 1, next(f) + mo = re.match(r'static label labels\[(\d+)\] = {$', line) + assert mo, (lineno, line) + nlabels = int(mo.group(1)) + for i in range(nlabels): + lineno, line = lineno + 1, next(f) + mo = re.match(r'\s+{(\d+), (0|"\w+")},$', line) + assert mo, (lineno, line) + x, y = mo.groups() + x = int(x) + if y == '0': + y = None + else: + y = eval(y) + labels.append((x, y)) + lineno, line = lineno + 1, next(f) + assert line == '};\n', (lineno, line) + self.labels = labels + + # Parse the grammar struct + lineno, line = lineno + 1, next(f) + assert line == 'grammar _PyParser_Grammar = {\n', (lineno, line) + lineno, line = lineno + 1, next(f) + mo = re.match(r'\s+(\d+),$', line) + assert mo, (lineno, line) + ndfas = int(mo.group(1)) + assert ndfas == len(self.dfas) + lineno, line = lineno + 1, next(f) + assert line == '\tdfas,\n', (lineno, line) + lineno, line = lineno + 1, next(f) + mo = re.match(r'\s+{(\d+), labels},$', line) + assert mo, (lineno, line) + nlabels = int(mo.group(1)) + assert nlabels == len(self.labels), (lineno, line) + lineno, line = lineno + 1, next(f) + mo = re.match(r'\s+(\d+)$', line) + assert mo, (lineno, line) + start = int(mo.group(1)) + assert start in self.number2symbol, (lineno, line) + self.start = start + lineno, line = lineno + 1, next(f) + assert line == '};\n', (lineno, line) + try: + lineno, line = lineno + 1, next(f) + except StopIteration: + pass + else: + assert 0, (lineno, line) + + def finish_off(self): + """Create additional useful structures. (Internal).""" + self.keywords = {} # map from keyword strings to arc labels + self.tokens = {} # map from numeric token values to arc labels + for ilabel, (type, value) in enumerate(self.labels): + if type == token.NAME and value is not None: + self.keywords[value] = ilabel + elif value is None: + self.tokens[type] = ilabel diff --git a/third_party/yapf_third_party/_ylib2to3/pgen2/driver.py b/third_party/yapf_third_party/_ylib2to3/pgen2/driver.py new file mode 100644 index 000000000..76b31a11c --- /dev/null +++ b/third_party/yapf_third_party/_ylib2to3/pgen2/driver.py @@ -0,0 +1,296 @@ +# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. + +# Modifications: +# Copyright 2006 Google, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. +"""Parser driver. + +This provides a high-level interface to parse a file into a syntax tree. + +""" + +__author__ = 'Guido van Rossum ' + +__all__ = ['Driver', 'load_grammar'] + +import io +import logging +import os +import pkgutil +import sys +# Python imports +from contextlib import contextmanager +from dataclasses import dataclass +from dataclasses import field +from pathlib import Path +from typing import Any +from typing import Iterator +from typing import List +from typing import Optional + +from platformdirs import user_cache_dir + +from yapf._version import __version__ as yapf_version + +# Pgen imports +from . import grammar +from . import parse +from . import pgen +from . import token +from . import tokenize + + +@dataclass +class ReleaseRange: + start: int + end: Optional[int] = None + tokens: List[Any] = field(default_factory=list) + + def lock(self) -> None: + total_eaten = len(self.tokens) + self.end = self.start + total_eaten + + +class TokenProxy: + + def __init__(self, generator: Any) -> None: + self._tokens = generator + self._counter = 0 + self._release_ranges: List[ReleaseRange] = [] + + @contextmanager + def release(self) -> Iterator['TokenProxy']: + release_range = ReleaseRange(self._counter) + self._release_ranges.append(release_range) + try: + yield self + finally: + # Lock the last release range to the final position that + # has been eaten. + release_range.lock() + + def eat(self, point: int) -> Any: + eaten_tokens = self._release_ranges[-1].tokens + if point < len(eaten_tokens): + return eaten_tokens[point] + else: + while point >= len(eaten_tokens): + token = next(self._tokens) + eaten_tokens.append(token) + return token + + def __iter__(self) -> 'TokenProxy': + return self + + def __next__(self) -> Any: + # If the current position is already compromised (looked up) + # return the eaten token, if not just go further on the given + # token producer. + for release_range in self._release_ranges: + assert release_range.end is not None + + start, end = release_range.start, release_range.end + if start <= self._counter < end: + token = release_range.tokens[self._counter - start] + break + else: + token = next(self._tokens) + self._counter += 1 + return token + + def can_advance(self, to: int) -> bool: + # Try to eat, fail if it can't. The eat operation is cached + # so there wont be any additional cost of eating here + try: + self.eat(to) + except StopIteration: + return False + else: + return True + + +class Driver(object): + + def __init__(self, grammar, convert=None, logger=None): + self.grammar = grammar + if logger is None: + logger = logging.getLogger() + self.logger = logger + self.convert = convert + + def parse_tokens(self, tokens, debug=False): + """Parse a series of tokens and return the syntax tree.""" + # XXX Move the prefix computation into a wrapper around tokenize. + p = parse.Parser(self.grammar, self.convert) + proxy = TokenProxy(tokens) + p.setup(proxy=proxy) + lineno = 1 + column = 0 + type = value = start = end = line_text = None + prefix = '' + for quintuple in proxy: + type, value, start, end, line_text = quintuple + if start != (lineno, column): + assert (lineno, column) <= start, ((lineno, column), start) + s_lineno, s_column = start + if lineno < s_lineno: + prefix += '\n' * (s_lineno - lineno) + lineno = s_lineno + column = 0 + if column < s_column: + prefix += line_text[column:s_column] + column = s_column + if type in (tokenize.COMMENT, tokenize.NL): + prefix += value + lineno, column = end + if value.endswith('\n'): + lineno += 1 + column = 0 + continue + if type == token.OP: + type = grammar.opmap[value] + if debug: + self.logger.debug('%s %r (prefix=%r)', token.tok_name[type], value, + prefix) + if p.addtoken(type, value, (prefix, start)): + if debug: + self.logger.debug('Stop.') + break + prefix = '' + lineno, column = end + if value.endswith('\n'): + lineno += 1 + column = 0 + else: + # We never broke out -- EOF is too soon (how can this happen???) + raise parse.ParseError('incomplete input', type, value, (prefix, start)) + return p.rootnode + + def parse_stream_raw(self, stream, debug=False): + """Parse a stream and return the syntax tree.""" + tokens = tokenize.generate_tokens(stream.readline) + return self.parse_tokens(tokens, debug) + + def parse_stream(self, stream, debug=False): + """Parse a stream and return the syntax tree.""" + return self.parse_stream_raw(stream, debug) + + def parse_file(self, filename, encoding=None, debug=False): + """Parse a file and return the syntax tree.""" + with io.open(filename, 'r', encoding=encoding) as stream: + return self.parse_stream(stream, debug) + + def parse_string(self, text, debug=False): + """Parse a string and return the syntax tree.""" + tokens = tokenize.generate_tokens(io.StringIO(text).readline) + return self.parse_tokens(tokens, debug) + + +def _generate_pickle_name(gt): + # type:(str) -> str + """Get the filepath to write a pickle file to + given the path of a grammar textfile. + + The returned filepath should be in a user-specific cache directory. + + Args: + gt (str): path to grammar text file + + Returns: + str: path to pickle file + """ + + grammar_textfile_name = os.path.basename(gt) + head, tail = os.path.splitext(grammar_textfile_name) + if tail == '.txt': + tail = '' + cache_dir = user_cache_dir( + appname='YAPF', appauthor='Google', version=yapf_version) + return cache_dir + os.sep + head + tail + '-py' + '.'.join( + map(str, sys.version_info)) + '.pickle' + + +def load_grammar(gt='Grammar.txt', + gp=None, + save=True, + force=False, + logger=None): + # type:(str, str | None, bool, bool, logging.Logger | None) -> grammar.Grammar + """Load the grammar (maybe from a pickle).""" + if logger is None: + logger = logging.getLogger() + gp = _generate_pickle_name(gt) if gp is None else gp + grammar_text = gt + try: + newer = _newer(gp, gt) + except OSError as err: + logger.debug('OSError, could not check if newer: %s', err.args) + newer = True + if not os.path.exists(gt): + # Assume package data + gt_basename = os.path.basename(gt) + pd = pkgutil.get_data('yapf_third_party._ylib2to3', gt_basename) + if pd is None: + raise RuntimeError('Failed to load grammer %s from package' % gt_basename) + grammar_text = io.StringIO(pd.decode(encoding='utf-8')) + if force or not newer: + g = pgen.generate_grammar(grammar_text) + if save: + try: + Path(gp).parent.mkdir(parents=True, exist_ok=True) + g.dump(gp) + except OSError: + # Ignore error, caching is not vital. + pass + else: + g = grammar.Grammar() + g.load(gp) + return g + + +def _newer(a, b): + """Inquire whether file a was written since file b.""" + if not os.path.exists(a): + return False + if not os.path.exists(b): + return True + return os.path.getmtime(a) >= os.path.getmtime(b) + + +def load_packaged_grammar(package, grammar_source): + """Normally, loads a pickled grammar by doing + pkgutil.get_data(package, pickled_grammar) + where *pickled_grammar* is computed from *grammar_source* by adding the + Python version and using a ``.pickle`` extension. + + However, if *grammar_source* is an extant file, load_grammar(grammar_source) + is called instead. This facilitates using a packaged grammar file when needed + but preserves load_grammar's automatic regeneration behavior when possible. + + """ # noqa: E501 + if os.path.isfile(grammar_source): + return load_grammar(grammar_source) + pickled_name = _generate_pickle_name(os.path.basename(grammar_source)) + data = pkgutil.get_data(package, pickled_name) + g = grammar.Grammar() + g.loads(data) + return g + + +def main(*args): + """Main program, when run as a script: produce grammar pickle files. + + Calls load_grammar for each argument, a path to a grammar text file. + """ + if not args: + args = sys.argv[1:] + logging.basicConfig( + level=logging.INFO, stream=sys.stdout, format='%(message)s') + for gt in args: + load_grammar(gt, save=True, force=True) + return True + + +if __name__ == '__main__': + sys.exit(int(not main())) diff --git a/third_party/yapf_third_party/_ylib2to3/pgen2/grammar.py b/third_party/yapf_third_party/_ylib2to3/pgen2/grammar.py new file mode 100644 index 000000000..3825ce7e5 --- /dev/null +++ b/third_party/yapf_third_party/_ylib2to3/pgen2/grammar.py @@ -0,0 +1,221 @@ +# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. +"""This module defines the data structures used to represent a grammar. + +These are a bit arcane because they are derived from the data +structures used by Python's 'pgen' parser generator. + +There's also a table here mapping operators to their names in the +token module; the Python tokenize module reports all operators as the +fallback token code OP, but the parser needs the actual token code. + +""" + +# Python imports +import os +import pickle +import tempfile + +# Local imports +from . import token + + +class Grammar(object): + """Pgen parsing tables conversion class. + + Once initialized, this class supplies the grammar tables for the + parsing engine implemented by parse.py. The parsing engine + accesses the instance variables directly. The class here does not + provide initialization of the tables; several subclasses exist to + do this (see the conv and pgen modules). + + The load() method reads the tables from a pickle file, which is + much faster than the other ways offered by subclasses. The pickle + file is written by calling dump() (after loading the grammar + tables using a subclass). The report() method prints a readable + representation of the tables to stdout, for debugging. + + The instance variables are as follows: + + symbol2number -- a dict mapping symbol names to numbers. Symbol + numbers are always 256 or higher, to distinguish + them from token numbers, which are between 0 and + 255 (inclusive). + + number2symbol -- a dict mapping numbers to symbol names; + these two are each other's inverse. + + states -- a list of DFAs, where each DFA is a list of + states, each state is a list of arcs, and each + arc is a (i, j) pair where i is a label and j is + a state number. The DFA number is the index into + this list. (This name is slightly confusing.) + Final states are represented by a special arc of + the form (0, j) where j is its own state number. + + dfas -- a dict mapping symbol numbers to (DFA, first) + pairs, where DFA is an item from the states list + above, and first is a set of tokens that can + begin this grammar rule (represented by a dict + whose values are always 1). + + labels -- a list of (x, y) pairs where x is either a token + number or a symbol number, and y is either None + or a string; the strings are keywords. The label + number is the index in this list; label numbers + are used to mark state transitions (arcs) in the + DFAs. + + start -- the number of the grammar's start symbol. + + keywords -- a dict mapping keyword strings to arc labels. + + tokens -- a dict mapping token numbers to arc labels. + + """ + + def __init__(self): + self.symbol2number = {} + self.number2symbol = {} + self.states = [] + self.dfas = {} + self.labels = [(0, 'EMPTY')] + self.keywords = {} + self.soft_keywords = {} + self.tokens = {} + self.symbol2label = {} + self.start = 256 + + def dump(self, filename): + """Dump the grammar tables to a pickle file.""" + # NOTE: + # - We're writing a tempfile first so that there is no chance + # for someone to read a half-written file from this very spot + # while we're were not done writing. + # - We're using ``os.rename`` to sure not copy data around (which + # would get us back to square one with a reading-half-written file + # race condition). + # - We're making the tempfile go to the same directory as the eventual + # target ``filename`` so that there is no chance of failing from + # cross-file-system renames in ``os.rename``. + # - We're using the same prefix and suffix for the tempfile so if we + # ever have to leave a tempfile around for failure of deletion, + # it will have a reasonable filename extension and its name will help + # explain is nature. + tempfile_dir = os.path.dirname(filename) + tempfile_prefix, tempfile_suffix = os.path.splitext(filename) + with tempfile.NamedTemporaryFile( + mode='wb', + suffix=tempfile_suffix, + prefix=tempfile_prefix, + dir=tempfile_dir, + delete=False) as f: + pickle.dump(self.__dict__, f.file, pickle.HIGHEST_PROTOCOL) + try: + os.rename(f.name, filename) + except OSError: + # This makes sure that we do not leave the tempfile around + # unless we have to... + try: + os.remove(f.name) + except OSError: + pass + raise + + def load(self, filename): + """Load the grammar tables from a pickle file.""" + with open(filename, 'rb') as f: + d = pickle.load(f) + self.__dict__.update(d) + + def loads(self, pkl): + """Load the grammar tables from a pickle bytes object.""" + self.__dict__.update(pickle.loads(pkl)) + + def copy(self): + """ + Copy the grammar. + """ + new = self.__class__() + for dict_attr in ('symbol2number', 'number2symbol', 'dfas', 'keywords', + 'soft_keywords', 'tokens', 'symbol2label'): + setattr(new, dict_attr, getattr(self, dict_attr).copy()) + new.labels = self.labels[:] + new.states = self.states[:] + new.start = self.start + return new + + def report(self): + """Dump the grammar tables to standard output, for debugging.""" + from pprint import pprint + print('s2n') + pprint(self.symbol2number) + print('n2s') + pprint(self.number2symbol) + print('states') + pprint(self.states) + print('dfas') + pprint(self.dfas) + print('labels') + pprint(self.labels) + print('start', self.start) + + +# Map from operator to number (since tokenize doesn't do this) + +opmap_raw = """ +( LPAR +) RPAR +[ LSQB +] RSQB +: COLON +, COMMA +; SEMI ++ PLUS +- MINUS +* STAR +/ SLASH +| VBAR +& AMPER +< LESS +> GREATER += EQUAL +. DOT +% PERCENT +` BACKQUOTE +{ LBRACE +} RBRACE +@ AT +@= ATEQUAL +== EQEQUAL +!= NOTEQUAL +<> NOTEQUAL +<= LESSEQUAL +>= GREATEREQUAL +~ TILDE +^ CIRCUMFLEX +<< LEFTSHIFT +>> RIGHTSHIFT +** DOUBLESTAR ++= PLUSEQUAL +-= MINEQUAL +*= STAREQUAL +/= SLASHEQUAL +%= PERCENTEQUAL +&= AMPEREQUAL +|= VBAREQUAL +^= CIRCUMFLEXEQUAL +<<= LEFTSHIFTEQUAL +>>= RIGHTSHIFTEQUAL +**= DOUBLESTAREQUAL +// DOUBLESLASH +//= DOUBLESLASHEQUAL +-> RARROW +:= COLONEQUAL +""" + +opmap = {} +for line in opmap_raw.splitlines(): + if line: + op, name = line.split() + opmap[op] = getattr(token, name) diff --git a/third_party/yapf_third_party/_ylib2to3/pgen2/literals.py b/third_party/yapf_third_party/_ylib2to3/pgen2/literals.py new file mode 100644 index 000000000..62d1d2681 --- /dev/null +++ b/third_party/yapf_third_party/_ylib2to3/pgen2/literals.py @@ -0,0 +1,64 @@ +# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. +"""Safely evaluate Python string literals without using eval().""" + +import re + +simple_escapes = { + 'a': '\a', + 'b': '\b', + 'f': '\f', + 'n': '\n', + 'r': '\r', + 't': '\t', + 'v': '\v', + "'": "'", + '"': '"', + '\\': '\\' +} + + +def escape(m): + all, tail = m.group(0, 1) + assert all.startswith('\\') + esc = simple_escapes.get(tail) + if esc is not None: + return esc + if tail.startswith('x'): + hexes = tail[1:] + if len(hexes) < 2: + raise ValueError("invalid hex string escape ('\\%s')" % tail) + try: + i = int(hexes, 16) + except ValueError: + raise ValueError("invalid hex string escape ('\\%s')" % tail) from None + else: + try: + i = int(tail, 8) + except ValueError: + raise ValueError("invalid octal string escape ('\\%s')" % tail) from None + return chr(i) + + +def evalString(s): + assert s.startswith("'") or s.startswith('"'), repr(s[:1]) + q = s[0] + if s[:3] == q * 3: + q = q * 3 + assert s.endswith(q), repr(s[-len(q):]) + assert len(s) >= 2 * len(q) + s = s[len(q):-len(q)] + return re.sub(r"\\(\'|\"|\\|[abfnrtv]|x.{0,2}|[0-7]{1,3})", escape, s) + + +def test(): + for i in range(256): + c = chr(i) + s = repr(c) + e = evalString(s) + if e != c: + print(i, c, s, e) + + +if __name__ == '__main__': + test() diff --git a/third_party/yapf_third_party/_ylib2to3/pgen2/parse.py b/third_party/yapf_third_party/_ylib2to3/pgen2/parse.py new file mode 100644 index 000000000..924b0e74b --- /dev/null +++ b/third_party/yapf_third_party/_ylib2to3/pgen2/parse.py @@ -0,0 +1,378 @@ +# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. +"""Parser engine for the grammar tables generated by pgen. + +The grammar table must be loaded first. + +See Parser/parser.c in the Python distribution for additional info on +how this parsing engine works. + +""" +from contextlib import contextmanager +from typing import Any +from typing import Callable +from typing import Dict +from typing import Iterator +from typing import List +from typing import Optional +from typing import Set +from typing import Text +from typing import Tuple +from typing import cast + +from ..pytree import Context +from ..pytree import RawNode +from ..pytree import convert +# Local imports +from . import grammar +from . import token +from . import tokenize + +DFA = List[List[Tuple[int, int]]] +DFAS = Tuple[DFA, Dict[int, int]] + +# A placeholder node, used when parser is backtracking. +DUMMY_NODE = (-1, None, None, None) + + +def stack_copy( + stack: List[Tuple[DFAS, int, RawNode]]) -> List[Tuple[DFAS, int, RawNode]]: + """Nodeless stack copy.""" + return [(dfa, label, DUMMY_NODE) for dfa, label, _ in stack] + + +class Recorder: + + def __init__(self, parser: 'Parser', ilabels: List[int], + context: Context) -> None: + self.parser = parser + self._ilabels = ilabels + self.context = context # not really matter + + self._dead_ilabels: Set[int] = set() + self._start_point = self.parser.stack + self._points = {ilabel: stack_copy(self._start_point) for ilabel in ilabels} + + @property + def ilabels(self) -> Set[int]: + return self._dead_ilabels.symmetric_difference(self._ilabels) + + @contextmanager + def switch_to(self, ilabel: int) -> Iterator[None]: + with self.backtrack(): + self.parser.stack = self._points[ilabel] + try: + yield + except ParseError: + self._dead_ilabels.add(ilabel) + finally: + self.parser.stack = self._start_point + + @contextmanager + def backtrack(self) -> Iterator[None]: + """ + Use the node-level invariant ones for basic parsing operations (push/pop/shift). + These still will operate on the stack; but they won't create any new nodes, or + modify the contents of any other existing nodes. + This saves us a ton of time when we are backtracking, since we + want to restore to the initial state as quick as possible, which + can only be done by having as little mutatations as possible. + """ # noqa: E501 + is_backtracking = self.parser.is_backtracking + try: + self.parser.is_backtracking = True + yield + finally: + self.parser.is_backtracking = is_backtracking + + def add_token(self, tok_type: int, tok_val: Text, raw: bool = False) -> None: + func: Callable[..., Any] + if raw: + func = self.parser._addtoken + else: + func = self.parser.addtoken + + for ilabel in self.ilabels: + with self.switch_to(ilabel): + args = [tok_type, tok_val, self.context] + if raw: + args.insert(0, ilabel) + func(*args) + + def determine_route(self, + value: Text = None, + force: bool = False) -> Optional[int]: + alive_ilabels = self.ilabels + if len(alive_ilabels) == 0: + *_, most_successful_ilabel = self._dead_ilabels + raise ParseError('bad input', most_successful_ilabel, value, self.context) + + ilabel, *rest = alive_ilabels + if force or not rest: + return ilabel + else: + return None + + +class ParseError(Exception): + """Exception to signal the parser is stuck.""" + + def __init__(self, msg, type, value, context): + Exception.__init__( + self, '%s: type=%r, value=%r, context=%r' % (msg, type, value, context)) + self.msg = msg + self.type = type + self.value = value + self.context = context + + def __reduce__(self): + return type(self), (self.msg, self.type, self.value, self.context) + + +class Parser(object): + """Parser engine. + + The proper usage sequence is: + + p = Parser(grammar, [converter]) # create instance + p.setup([start]) # prepare for parsing + : + if p.addtoken(...): # parse a token; may raise ParseError + break + root = p.rootnode # root of abstract syntax tree + + A Parser instance may be reused by calling setup() repeatedly. + + A Parser instance contains state pertaining to the current token + sequence, and should not be used concurrently by different threads + to parse separate token sequences. + + See driver.py for how to get input tokens by tokenizing a file or + string. + + Parsing is complete when addtoken() returns True; the root of the + abstract syntax tree can then be retrieved from the rootnode + instance variable. When a syntax error occurs, addtoken() raises + the ParseError exception. There is no error recovery; the parser + cannot be used after a syntax error was reported (but it can be + reinitialized by calling setup()). + + """ + + def __init__(self, grammar, convert=None): + """Constructor. + + The grammar argument is a grammar.Grammar instance; see the + grammar module for more information. + + The parser is not ready yet for parsing; you must call the + setup() method to get it started. + + The optional convert argument is a function mapping concrete + syntax tree nodes to abstract syntax tree nodes. If not + given, no conversion is done and the syntax tree produced is + the concrete syntax tree. If given, it must be a function of + two arguments, the first being the grammar (a grammar.Grammar + instance), and the second being the concrete syntax tree node + to be converted. The syntax tree is converted from the bottom + up. + + A concrete syntax tree node is a (type, value, context, nodes) + tuple, where type is the node type (a token or symbol number), + value is None for symbols and a string for tokens, context is + None or an opaque value used for error reporting (typically a + (lineno, offset) pair), and nodes is a list of children for + symbols, and None for tokens. + + An abstract syntax tree node may be anything; this is entirely + up to the converter function. + + """ + self.grammar = grammar + self.convert = convert or (lambda grammar, node: node) + self.is_backtracking = False + + def setup(self, proxy, start=None): + """Prepare for parsing. + + This *must* be called before starting to parse. + + The optional argument is an alternative start symbol; it + defaults to the grammar's start symbol. + + You can use a Parser instance to parse any number of programs; + each time you call setup() the parser is reset to an initial + state determined by the (implicit or explicit) start symbol. + + """ + if start is None: + start = self.grammar.start + # Each stack entry is a tuple: (dfa, state, node). + # A node is a tuple: (type, value, context, children), + # where children is a list of nodes or None, and context may be None. + newnode = (start, None, None, []) + stackentry = (self.grammar.dfas[start], 0, newnode) + self.stack = [stackentry] + self.rootnode = None + self.used_names = set() # Aliased to self.rootnode.used_names in pop() + self.proxy = proxy + + def addtoken(self, type, value, context): + """Add a token; return True iff this is the end of the program.""" + # Map from token to label + ilabels = self.classify(type, value, context) + assert len(ilabels) >= 1 + + # If we have only one state to advance, we'll directly + # take it as is. + if len(ilabels) == 1: + [ilabel] = ilabels + return self._addtoken(ilabel, type, value, context) + + # If there are multiple states which we can advance (only + # happen under soft-keywords), then we will try all of them + # in parallel and as soon as one state can reach further than + # the rest, we'll choose that one. This is a pretty hacky + # and hopefully temporary algorithm. + # + # For a more detailed explanation, check out this post: + # https://tree.science/what-the-backtracking.html + + with self.proxy.release() as proxy: + counter, force = 0, False + recorder = Recorder(self, ilabels, context) + recorder.add_token(type, value, raw=True) + + next_token_value = value + while recorder.determine_route(next_token_value) is None: + if not proxy.can_advance(counter): + force = True + break + + next_token_type, next_token_value, *_ = proxy.eat(counter) + if next_token_type in (tokenize.COMMENT, tokenize.NL): + counter += 1 + continue + + if next_token_type == tokenize.OP: + next_token_type = grammar.opmap[next_token_value] + + recorder.add_token(next_token_type, next_token_value) + counter += 1 + + ilabel = cast(int, + recorder.determine_route(next_token_value, force=force)) + assert ilabel is not None + + return self._addtoken(ilabel, type, value, context) + + def _addtoken(self, ilabel: int, type: int, value: Text, + context: Context) -> bool: + # Loop until the token is shifted; may raise exceptions + while True: + dfa, state, node = self.stack[-1] + states, first = dfa + arcs = states[state] + # Look for a state with this label + for i, newstate in arcs: + t = self.grammar.labels[i][0] + if t >= 256: + # See if it's a symbol and if we're in its first set + itsdfa = self.grammar.dfas[t] + itsstates, itsfirst = itsdfa + if ilabel in itsfirst: + # Push a symbol + self.push(t, itsdfa, newstate, context) + break # To continue the outer while loop + + elif ilabel == i: + # Look it up in the list of labels + # Shift a token; we're done with it + self.shift(type, value, newstate, context) + # Pop while we are in an accept-only state + state = newstate + while states[state] == [(0, state)]: + self.pop() + if not self.stack: + # Done parsing! + return True + dfa, state, node = self.stack[-1] + states, first = dfa + # Done with this token + return False + + else: + if (0, state) in arcs: + # An accepting state, pop it and try something else + self.pop() + if not self.stack: + # Done parsing, but another token is input + raise ParseError('too much input', type, value, context) + else: + # No success finding a transition + raise ParseError('bad input', type, value, context) + + def classify(self, type, value, context): + """Turn a token into a label. (Internal) + + Depending on whether the value is a soft-keyword or not, + this function may return multiple labels to choose from.""" + if type == token.NAME: + # Keep a listing of all used names + self.used_names.add(value) + # Check for reserved words + if value in self.grammar.keywords: + return [self.grammar.keywords[value]] + elif value in self.grammar.soft_keywords: + assert type in self.grammar.tokens + return [ + self.grammar.soft_keywords[value], + self.grammar.tokens[type], + ] + + ilabel = self.grammar.tokens.get(type) + if ilabel is None: + raise ParseError('bad token', type, value, context) + return [ilabel] + + def shift(self, type: int, value: Text, newstate: int, + context: Context) -> None: + """Shift a token. (Internal)""" + if self.is_backtracking: + dfa, state, _ = self.stack[-1] + self.stack[-1] = (dfa, newstate, DUMMY_NODE) + else: + dfa, state, node = self.stack[-1] + rawnode: RawNode = (type, value, context, None) + newnode = convert(self.grammar, rawnode) + assert node[-1] is not None + node[-1].append(newnode) + self.stack[-1] = (dfa, newstate, node) + + def push(self, type: int, newdfa: DFAS, newstate: int, + context: Context) -> None: + """Push a nonterminal. (Internal)""" + if self.is_backtracking: + dfa, state, _ = self.stack[-1] + self.stack[-1] = (dfa, newstate, DUMMY_NODE) + self.stack.append((newdfa, 0, DUMMY_NODE)) + else: + dfa, state, node = self.stack[-1] + newnode: RawNode = (type, None, context, []) + self.stack[-1] = (dfa, newstate, node) + self.stack.append((newdfa, 0, newnode)) + + def pop(self) -> None: + """Pop a nonterminal. (Internal)""" + if self.is_backtracking: + self.stack.pop() + else: + popdfa, popstate, popnode = self.stack.pop() + newnode = convert(self.grammar, popnode) + if self.stack: + dfa, state, node = self.stack[-1] + assert node[-1] is not None + node[-1].append(newnode) + else: + self.rootnode = newnode + self.rootnode.used_names = self.used_names diff --git a/third_party/yapf_third_party/_ylib2to3/pgen2/pgen.py b/third_party/yapf_third_party/_ylib2to3/pgen2/pgen.py new file mode 100644 index 000000000..6b96478ad --- /dev/null +++ b/third_party/yapf_third_party/_ylib2to3/pgen2/pgen.py @@ -0,0 +1,409 @@ +# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. + +# Pgen imports +from io import StringIO + +from . import grammar +from . import token +from . import tokenize + + +class PgenGrammar(grammar.Grammar): + pass + + +class ParserGenerator(object): + + def __init__(self, filename=None, stream=None): + close_stream = None + if filename is None and stream is None: + raise RuntimeError( + 'Either a filename or a stream is expected, both were none') + if stream is None: + stream = open(filename, encoding='utf-8') + close_stream = stream.close + self.filename = filename + self.stream = stream + self.generator = tokenize.generate_tokens(stream.readline) + self.gettoken() # Initialize lookahead + self.dfas, self.startsymbol = self.parse() + if close_stream is not None: + close_stream() + self.first = {} # map from symbol name to set of tokens + self.addfirstsets() + + def make_grammar(self): + c = PgenGrammar() + names = list(self.dfas.keys()) + names.sort() + names.remove(self.startsymbol) + names.insert(0, self.startsymbol) + for name in names: + i = 256 + len(c.symbol2number) + c.symbol2number[name] = i + c.number2symbol[i] = name + for name in names: + dfa = self.dfas[name] + states = [] + for state in dfa: + arcs = [] + for label, next in sorted(state.arcs.items()): + arcs.append((self.make_label(c, label), dfa.index(next))) + if state.isfinal: + arcs.append((0, dfa.index(state))) + states.append(arcs) + c.states.append(states) + c.dfas[c.symbol2number[name]] = (states, self.make_first(c, name)) + c.start = c.symbol2number[self.startsymbol] + return c + + def make_first(self, c, name): + rawfirst = self.first[name] + first = {} + for label in sorted(rawfirst): + ilabel = self.make_label(c, label) + # assert ilabel not in first # XXX failed on <> ... != + first[ilabel] = 1 + return first + + def make_label(self, c, label): + # XXX Maybe this should be a method on a subclass of converter? + ilabel = len(c.labels) + if label[0].isalpha(): + # Either a symbol name or a named token + if label in c.symbol2number: + # A symbol name (a non-terminal) + if label in c.symbol2label: + return c.symbol2label[label] + else: + c.labels.append((c.symbol2number[label], None)) + c.symbol2label[label] = ilabel + return ilabel + else: + # A named token (NAME, NUMBER, STRING) + itoken = getattr(token, label, None) + assert isinstance(itoken, int), label + assert itoken in token.tok_name, label + if itoken in c.tokens: + return c.tokens[itoken] + else: + c.labels.append((itoken, None)) + c.tokens[itoken] = ilabel + return ilabel + else: + # Either a keyword or an operator + assert label[0] in ('"', "'"), label + value = eval(label) + if value[0].isalpha(): + if label[0] == '"': + keywords = c.soft_keywords + else: + keywords = c.keywords + + # A keyword + if value in keywords: + return keywords[value] + else: + c.labels.append((token.NAME, value)) + keywords[value] = ilabel + return ilabel + else: + # An operator (any non-numeric token) + itoken = grammar.opmap[value] # Fails if unknown token + if itoken in c.tokens: + return c.tokens[itoken] + else: + c.labels.append((itoken, None)) + c.tokens[itoken] = ilabel + return ilabel + + def addfirstsets(self): + names = list(self.dfas.keys()) + names.sort() + for name in names: + if name not in self.first: + self.calcfirst(name) + # print name, self.first[name].keys() + + def calcfirst(self, name): + dfa = self.dfas[name] + self.first[name] = None # dummy to detect left recursion + state = dfa[0] + totalset = {} + overlapcheck = {} + for label, next in state.arcs.items(): + if label in self.dfas: + if label in self.first: + fset = self.first[label] + if fset is None: + raise ValueError('recursion for rule %r' % name) + else: + self.calcfirst(label) + fset = self.first[label] + totalset.update(fset) + overlapcheck[label] = fset + else: + totalset[label] = 1 + overlapcheck[label] = {label: 1} + inverse = {} + for label, itsfirst in overlapcheck.items(): + for symbol in itsfirst: + if symbol in inverse: + raise ValueError('rule %s is ambiguous; %s is in the' + ' first sets of %s as well as %s' % + (name, symbol, label, inverse[symbol])) + inverse[symbol] = label + self.first[name] = totalset + + def parse(self): + dfas = {} + startsymbol = None + # MSTART: (NEWLINE | RULE)* ENDMARKER + while self.type != token.ENDMARKER: + while self.type == token.NEWLINE: + self.gettoken() + # RULE: NAME ':' RHS NEWLINE + name = self.expect(token.NAME) + self.expect(token.OP, ':') + a, z = self.parse_rhs() + self.expect(token.NEWLINE) + # self.dump_nfa(name, a, z) + dfa = self.make_dfa(a, z) + # self.dump_dfa(name, dfa) + self.simplify_dfa(dfa) + dfas[name] = dfa + # print name, oldlen, newlen + if startsymbol is None: + startsymbol = name + return dfas, startsymbol + + def make_dfa(self, start, finish): + # To turn an NFA into a DFA, we define the states of the DFA + # to correspond to *sets* of states of the NFA. Then do some + # state reduction. Let's represent sets as dicts with 1 for + # values. + assert isinstance(start, NFAState) + assert isinstance(finish, NFAState) + + def closure(state): + base = {} + addclosure(state, base) + return base + + def addclosure(state, base): + assert isinstance(state, NFAState) + if state in base: + return + base[state] = 1 + for label, next in state.arcs: + if label is None: + addclosure(next, base) + + states = [DFAState(closure(start), finish)] + for state in states: # NB states grows while we're iterating + arcs = {} + for nfastate in state.nfaset: + for label, next in nfastate.arcs: + if label is not None: + addclosure(next, arcs.setdefault(label, {})) + for label, nfaset in sorted(arcs.items()): + for st in states: + if st.nfaset == nfaset: + break + else: + st = DFAState(nfaset, finish) + states.append(st) + state.addarc(st, label) + return states # List of DFAState instances; first one is start + + def dump_nfa(self, name, start, finish): + print('Dump of NFA for', name) + todo = [start] + for i, state in enumerate(todo): + print(' State', i, state is finish and '(final)' or '') + for label, next in state.arcs: + if next in todo: + j = todo.index(next) + else: + j = len(todo) + todo.append(next) + if label is None: + print(' -> %d' % j) + else: + print(' %s -> %d' % (label, j)) + + def dump_dfa(self, name, dfa): + print('Dump of DFA for', name) + for i, state in enumerate(dfa): + print(' State', i, state.isfinal and '(final)' or '') + for label, next in sorted(state.arcs.items()): + print(' %s -> %d' % (label, dfa.index(next))) + + def simplify_dfa(self, dfa): + # This is not theoretically optimal, but works well enough. + # Algorithm: repeatedly look for two states that have the same + # set of arcs (same labels pointing to the same nodes) and + # unify them, until things stop changing. + + # dfa is a list of DFAState instances + changes = True + while changes: + changes = False + for i, state_i in enumerate(dfa): + for j in range(i + 1, len(dfa)): + state_j = dfa[j] + if state_i == state_j: + # print " unify", i, j + del dfa[j] + for state in dfa: + state.unifystate(state_j, state_i) + changes = True + break + + def parse_rhs(self): + # RHS: ALT ('|' ALT)* + a, z = self.parse_alt() + if self.value != '|': + return a, z + else: + aa = NFAState() + zz = NFAState() + aa.addarc(a) + z.addarc(zz) + while self.value == '|': + self.gettoken() + a, z = self.parse_alt() + aa.addarc(a) + z.addarc(zz) + return aa, zz + + def parse_alt(self): + # ALT: ITEM+ + a, b = self.parse_item() + while (self.value in ('(', '[') or self.type in (token.NAME, token.STRING)): + c, d = self.parse_item() + b.addarc(c) + b = d + return a, b + + def parse_item(self): + # ITEM: '[' RHS ']' | ATOM ['+' | '*'] + if self.value == '[': + self.gettoken() + a, z = self.parse_rhs() + self.expect(token.OP, ']') + a.addarc(z) + return a, z + else: + a, z = self.parse_atom() + value = self.value + if value not in ('+', '*'): + return a, z + self.gettoken() + z.addarc(a) + if value == '+': + return a, z + else: + return a, a + + def parse_atom(self): + # ATOM: '(' RHS ')' | NAME | STRING + if self.value == '(': + self.gettoken() + a, z = self.parse_rhs() + self.expect(token.OP, ')') + return a, z + elif self.type in (token.NAME, token.STRING): + a = NFAState() + z = NFAState() + a.addarc(z, self.value) + self.gettoken() + return a, z + else: + self.raise_error('expected (...) or NAME or STRING, got %s/%s', self.type, + self.value) + + def expect(self, type, value=None): + if self.type != type or (value is not None and self.value != value): + self.raise_error('expected %s/%s, got %s/%s', type, value, self.type, + self.value) + value = self.value + self.gettoken() + return value + + def gettoken(self): + tup = next(self.generator) + while tup[0] in (tokenize.COMMENT, tokenize.NL): + tup = next(self.generator) + self.type, self.value, self.begin, self.end, self.line = tup + # print token.tok_name[self.type], repr(self.value) + + def raise_error(self, msg, *args): + if args: + try: + msg = msg % args + except Exception: + msg = ' '.join([msg] + list(map(str, args))) + raise SyntaxError(msg, (self.filename, self.end[0], self.end[1], self.line)) + + +class NFAState(object): + + def __init__(self): + self.arcs = [] # list of (label, NFAState) pairs + + def addarc(self, next, label=None): + assert label is None or isinstance(label, str) + assert isinstance(next, NFAState) + self.arcs.append((label, next)) + + +class DFAState(object): + + def __init__(self, nfaset, final): + assert isinstance(nfaset, dict) + assert isinstance(next(iter(nfaset)), NFAState) + assert isinstance(final, NFAState) + self.nfaset = nfaset + self.isfinal = final in nfaset + self.arcs = {} # map from label to DFAState + + def addarc(self, next, label): + assert isinstance(label, str) + assert label not in self.arcs + assert isinstance(next, DFAState) + self.arcs[label] = next + + def unifystate(self, old, new): + for label, next in self.arcs.items(): + if next is old: + self.arcs[label] = new + + def __eq__(self, other): + # Equality test -- ignore the nfaset instance variable + assert isinstance(other, DFAState) + if self.isfinal != other.isfinal: + return False + # Can't just return self.arcs == other.arcs, because that + # would invoke this method recursively, with cycles... + if len(self.arcs) != len(other.arcs): + return False + for label, next in self.arcs.items(): + if next is not other.arcs.get(label): + return False + return True + + __hash__ = None # For Py3 compatibility. + + +def generate_grammar(filename_or_stream='Grammar.txt'): + # type:(str | StringIO) -> PgenGrammar + if isinstance(filename_or_stream, str): + p = ParserGenerator(filename_or_stream) + elif isinstance(filename_or_stream, StringIO): + p = ParserGenerator(stream=filename_or_stream) + else: + raise NotImplementedError('Type %s not implemented' % + type(filename_or_stream)) + return p.make_grammar() diff --git a/third_party/yapf_third_party/_ylib2to3/pgen2/token.py b/third_party/yapf_third_party/_ylib2to3/pgen2/token.py new file mode 100644 index 000000000..fbcd15525 --- /dev/null +++ b/third_party/yapf_third_party/_ylib2to3/pgen2/token.py @@ -0,0 +1,88 @@ +#! /usr/bin/env python3 +"""Token constants (from "token.h").""" + +# Taken from Python (r53757) and modified to include some tokens +# originally monkeypatched in by pgen2.tokenize + +# --start constants-- +ENDMARKER = 0 +NAME = 1 +NUMBER = 2 +STRING = 3 +NEWLINE = 4 +INDENT = 5 +DEDENT = 6 +LPAR = 7 +RPAR = 8 +LSQB = 9 +RSQB = 10 +COLON = 11 +COMMA = 12 +SEMI = 13 +PLUS = 14 +MINUS = 15 +STAR = 16 +SLASH = 17 +VBAR = 18 +AMPER = 19 +LESS = 20 +GREATER = 21 +EQUAL = 22 +DOT = 23 +PERCENT = 24 +BACKQUOTE = 25 +LBRACE = 26 +RBRACE = 27 +EQEQUAL = 28 +NOTEQUAL = 29 +LESSEQUAL = 30 +GREATEREQUAL = 31 +TILDE = 32 +CIRCUMFLEX = 33 +LEFTSHIFT = 34 +RIGHTSHIFT = 35 +DOUBLESTAR = 36 +PLUSEQUAL = 37 +MINEQUAL = 38 +STAREQUAL = 39 +SLASHEQUAL = 40 +PERCENTEQUAL = 41 +AMPEREQUAL = 42 +VBAREQUAL = 43 +CIRCUMFLEXEQUAL = 44 +LEFTSHIFTEQUAL = 45 +RIGHTSHIFTEQUAL = 46 +DOUBLESTAREQUAL = 47 +DOUBLESLASH = 48 +DOUBLESLASHEQUAL = 49 +AT = 50 +ATEQUAL = 51 +OP = 52 +COMMENT = 53 +NL = 54 +RARROW = 55 +AWAIT = 56 +ASYNC = 57 +ERRORTOKEN = 58 +COLONEQUAL = 59 +N_TOKENS = 60 +NT_OFFSET = 256 +# --end constants-- + +tok_name = { + _value: _name + for _name, _value in globals().copy().items() + if isinstance(_value, int) +} + + +def ISTERMINAL(x): + return x < NT_OFFSET + + +def ISNONTERMINAL(x): + return x >= NT_OFFSET + + +def ISEOF(x): + return x == ENDMARKER diff --git a/third_party/yapf_third_party/_ylib2to3/pgen2/tokenize.py b/third_party/yapf_third_party/_ylib2to3/pgen2/tokenize.py new file mode 100644 index 000000000..dda83296f --- /dev/null +++ b/third_party/yapf_third_party/_ylib2to3/pgen2/tokenize.py @@ -0,0 +1,611 @@ +# Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006 Python Software Foundation. +# All rights reserved. +"""Tokenization help for Python programs. + +generate_tokens(readline) is a generator that breaks a stream of +text into Python tokens. It accepts a readline-like method which is called +repeatedly to get the next line of input (or "" for EOF). It generates +5-tuples with these members: + + the token type (see token.py) + the token (a string) + the starting (row, column) indices of the token (a 2-tuple of ints) + the ending (row, column) indices of the token (a 2-tuple of ints) + the original line (string) + +It is designed to match the working of the Python tokenizer exactly, except +that it produces COMMENT tokens for comments and gives type OP for all +operators + +Older entry points + tokenize_loop(readline, tokeneater) + tokenize(readline, tokeneater=printtoken) +are the same, except instead of generating tokens, tokeneater is a callback +function to which the 5 fields described above are passed as 5 arguments, +each time a new token is found.""" + +__author__ = 'Ka-Ping Yee ' +__credits__ = \ + 'GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro' + +import re +import string +from codecs import BOM_UTF8 +from codecs import lookup + +from . import token +from .token import ASYNC +from .token import AWAIT +from .token import COMMENT +from .token import DEDENT +from .token import ENDMARKER +from .token import ERRORTOKEN +from .token import INDENT +from .token import NAME +from .token import NEWLINE +from .token import NL +from .token import NUMBER +from .token import OP +from .token import STRING +from .token import tok_name + +__all__ = [x for x in dir(token) if x[0] != '_' + ] + ['tokenize', 'generate_tokens', 'untokenize'] +del token + +try: + bytes +except NameError: + # Support bytes type in Python <= 2.5, so 2to3 turns itself into + # valid Python 3 code. + bytes = str + + +def group(*choices): + return '(' + '|'.join(choices) + ')' + + +def any(*choices): + return group(*choices) + '*' + + +def maybe(*choices): + return group(*choices) + '?' + + +def _combinations(*l): # noqa: E741 + return set( + x + y for x in l for y in l + ('',) if x.casefold() != y.casefold()) + + +Whitespace = r'[ \f\t]*' +Comment = r'#[^\r\n]*' +Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment) +Name = r'\w+' + +Binnumber = r'0[bB]_?[01]+(?:_[01]+)*' +Hexnumber = r'0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?' +Octnumber = r'0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?' +Decnumber = group(r'[1-9]\d*(?:_\d+)*[lL]?', '0[lL]?') +Intnumber = group(Binnumber, Hexnumber, Octnumber, Decnumber) +Exponent = r'[eE][-+]?\d+(?:_\d+)*' +Pointfloat = group(r'\d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?', + r'\.\d+(?:_\d+)*') + maybe(Exponent) +Expfloat = r'\d+(?:_\d+)*' + Exponent +Floatnumber = group(Pointfloat, Expfloat) +Imagnumber = group(r'\d+(?:_\d+)*[jJ]', Floatnumber + r'[jJ]') +Number = group(Imagnumber, Floatnumber, Intnumber) + +# Tail end of ' string. +Single = r"[^'\\]*(?:\\.[^'\\]*)*'" +# Tail end of " string. +Double = r'[^"\\]*(?:\\.[^"\\]*)*"' +# Tail end of ''' string. +Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''" +# Tail end of """ string. +Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""' +_litprefix = r'(?:[uUrRbBfF]|[rR][fFbB]|[fFbBuU][rR])?' +Triple = group(_litprefix + "'''", _litprefix + '"""') +# Single-line ' or " string. +String = group(_litprefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'", + _litprefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*"') + +# Because of leftmost-then-longest match semantics, be sure to put the +# longest operators first (e.g., if = came before ==, == would get +# recognized as two instances of =). +Operator = group(r'\*\*=?', r'>>=?', r'<<=?', r'<>', r'!=', r'//=?', r'->', + r'[+\-*/%&@|^=<>]=?', r'~') + +Bracket = '[][(){}]' +Special = group(r'\r?\n', r':=', r'[:;.,`@]') +Funny = group(Operator, Bracket, Special) + +PlainToken = group(Number, Funny, String, Name) +Token = Ignore + PlainToken + +# First (or only) line of ' or " string. +ContStr = group( + _litprefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" + group("'", r'\\\r?\n'), + _litprefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' + group('"', r'\\\r?\n')) +PseudoExtras = group(r'\\\r?\n', Comment, Triple) +PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) + +tokenprog, pseudoprog, single3prog, double3prog = map( + re.compile, (Token, PseudoToken, Single3, Double3)) + +_strprefixes = ( + _combinations('r', 'R', 'f', 'F') | _combinations('r', 'R', 'b', 'B') + | {'u', 'U', 'ur', 'uR', 'Ur', 'UR'}) + +endprogs = { + "'": re.compile(Single), + '"': re.compile(Double), + "'''": single3prog, + '"""': double3prog, + **{ + f"{prefix}'''": single3prog for prefix in _strprefixes + }, + **{ + f'{prefix}"""': double3prog for prefix in _strprefixes + }, + **{ + prefix: None for prefix in _strprefixes + } +} + +triple_quoted = ({"'''", '"""'} | {f"{prefix}'''" for prefix in _strprefixes} + | {f'{prefix}"""' for prefix in _strprefixes}) +single_quoted = ({"'", '"'} | {f"{prefix}'" for prefix in _strprefixes} + | {f'{prefix}"' for prefix in _strprefixes}) + +tabsize = 8 + + +class TokenError(Exception): + pass + + +class StopTokenizing(Exception): + pass + + +def printtoken(type, token, xxx_todo_changeme, xxx_todo_changeme1, + line): # for testing + (srow, scol) = xxx_todo_changeme + (erow, ecol) = xxx_todo_changeme1 + print('%d,%d-%d,%d:\t%s\t%s' % + (srow, scol, erow, ecol, tok_name[type], repr(token))) + + +def tokenize(readline, tokeneater=printtoken): + """ + The tokenize() function accepts two parameters: one representing the + input stream, and one providing an output mechanism for tokenize(). + + The first parameter, readline, must be a callable object which provides + the same interface as the readline() method of built-in file objects. + Each call to the function should return one line of input as a string. + + The second parameter, tokeneater, must also be a callable object. It is + called once for each token, with five arguments, corresponding to the + tuples generated by generate_tokens(). + """ + try: + tokenize_loop(readline, tokeneater) + except StopTokenizing: + pass + + +# backwards compatible interface +def tokenize_loop(readline, tokeneater): + for token_info in generate_tokens(readline): + tokeneater(*token_info) + + +class Untokenizer: + + def __init__(self): + self.tokens = [] + self.prev_row = 1 + self.prev_col = 0 + + def add_whitespace(self, start): + row, col = start + assert row <= self.prev_row + col_offset = col - self.prev_col + if col_offset: + self.tokens.append(' ' * col_offset) + + def untokenize(self, iterable): + for t in iterable: + if len(t) == 2: + self.compat(t, iterable) + break + tok_type, token, start, end, line = t + self.add_whitespace(start) + self.tokens.append(token) + self.prev_row, self.prev_col = end + if tok_type in (NEWLINE, NL): + self.prev_row += 1 + self.prev_col = 0 + return ''.join(self.tokens) + + def compat(self, token, iterable): + startline = False + indents = [] + toks_append = self.tokens.append + toknum, tokval = token + if toknum in (NAME, NUMBER): + tokval += ' ' + if toknum in (NEWLINE, NL): + startline = True + for tok in iterable: + toknum, tokval = tok[:2] + + if toknum in (NAME, NUMBER, ASYNC, AWAIT): + tokval += ' ' + + if toknum == INDENT: + indents.append(tokval) + continue + elif toknum == DEDENT: + indents.pop() + continue + elif toknum in (NEWLINE, NL): + startline = True + elif startline and indents: + toks_append(indents[-1]) + startline = False + toks_append(tokval) + + +cookie_re = re.compile(r'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)', re.ASCII) +blank_re = re.compile(br'^[ \t\f]*(?:[#\r\n]|$)', re.ASCII) + + +def _get_normal_name(orig_enc): + """Imitates get_normal_name in tokenizer.c.""" + # Only care about the first 12 characters. + enc = orig_enc[:12].lower().replace('_', '-') + if enc == 'utf-8' or enc.startswith('utf-8-'): + return 'utf-8' + if enc in ('latin-1', 'iso-8859-1', 'iso-latin-1') or \ + enc.startswith(('latin-1-', 'iso-8859-1-', 'iso-latin-1-')): + return 'iso-8859-1' + return orig_enc + + +def detect_encoding(readline): + """ + The detect_encoding() function is used to detect the encoding that should + be used to decode a Python source file. It requires one argument, readline, + in the same way as the tokenize() generator. + + It will call readline a maximum of twice, and return the encoding used + (as a string) and a list of any lines (left as bytes) it has read + in. + + It detects the encoding from the presence of a utf-8 bom or an encoding + cookie as specified in pep-0263. If both a bom and a cookie are present, but + disagree, a SyntaxError will be raised. If the encoding cookie is an invalid + charset, raise a SyntaxError. Note that if a utf-8 bom is found, + 'utf-8-sig' is returned. + + If no encoding is specified, then the default of 'utf-8' will be returned. + """ + bom_found = False + encoding = None + default = 'utf-8' + + def read_or_stop(): + try: + return readline() + except StopIteration: + return bytes() + + def find_cookie(line): + try: + line_string = line.decode('ascii') + except UnicodeDecodeError: + return None + match = cookie_re.match(line_string) + if not match: + return None + encoding = _get_normal_name(match.group(1)) + try: + codec = lookup(encoding) + except LookupError: + # This behaviour mimics the Python interpreter + raise SyntaxError('unknown encoding: ' + encoding) + + if bom_found: + if codec.name != 'utf-8': + # This behaviour mimics the Python interpreter + raise SyntaxError('encoding problem: utf-8') + encoding += '-sig' + return encoding + + first = read_or_stop() + if first.startswith(BOM_UTF8): + bom_found = True + first = first[3:] + default = 'utf-8-sig' + if not first: + return default, [] + + encoding = find_cookie(first) + if encoding: + return encoding, [first] + if not blank_re.match(first): + return default, [first] + + second = read_or_stop() + if not second: + return default, [first] + + encoding = find_cookie(second) + if encoding: + return encoding, [first, second] + + return default, [first, second] + + +def untokenize(iterable): + """Transform tokens back into Python source code. + + Each element returned by the iterable must be a token sequence + with at least two elements, a token number and token value. If + only two tokens are passed, the resulting output is poor. + + Round-trip invariant for full input: + Untokenized source will match input source exactly + + Round-trip invariant for limited input: + # Output text will tokenize the back to the input + t1 = [tok[:2] for tok in generate_tokens(f.readline)] + newcode = untokenize(t1) + readline = iter(newcode.splitlines(1)).next + t2 = [tok[:2] for tokin generate_tokens(readline)] + assert t1 == t2 + """ + ut = Untokenizer() + return ut.untokenize(iterable) + + +def generate_tokens(readline): + """ + The generate_tokens() generator requires one argument, readline, which + must be a callable object which provides the same interface as the + readline() method of built-in file objects. Each call to the function + should return one line of input as a string. Alternately, readline + can be a callable function terminating with StopIteration: + readline = open(myfile).next # Example of alternate readline + + The generator produces 5-tuples with these members: the token type; the + token string; a 2-tuple (srow, scol) of ints specifying the row and + column where the token begins in the source; a 2-tuple (erow, ecol) of + ints specifying the row and column where the token ends in the source; + and the line on which the token was found. The line passed is the + physical line. + """ + strstart = '' + endprog = '' + lnum = parenlev = continued = 0 + contstr, needcont = '', 0 + contline = None + indents = [0] + + # 'stashed' and 'async_*' are used for async/await parsing + stashed = None + async_def = False + async_def_indent = 0 + async_def_nl = False + + while 1: # loop over lines in stream + try: + line = readline() + except StopIteration: + line = '' + lnum = lnum + 1 + pos, max = 0, len(line) + + if contstr: # continued string + if not line: + raise TokenError('EOF in multi-line string', strstart) + endmatch = endprog.match(line) + if endmatch: + pos = end = endmatch.end(0) + yield (STRING, contstr + line[:end], strstart, (lnum, end), + contline + line) + contstr, needcont = '', 0 + contline = None + elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n': + yield (ERRORTOKEN, contstr + line, strstart, (lnum, len(line)), + contline) + contstr = '' + contline = None + continue + else: + contstr = contstr + line + contline = contline + line + continue + + elif parenlev == 0 and not continued: # new statement + if not line: + break + column = 0 + while pos < max: # measure leading whitespace + if line[pos] == ' ': + column = column + 1 + elif line[pos] == '\t': + column = (column // tabsize + 1) * tabsize + elif line[pos] == '\f': + column = 0 + else: + break + pos = pos + 1 + if pos == max: + break + + if stashed: + yield stashed + stashed = None + + if line[pos] in '#\r\n': # skip comments or blank lines + if line[pos] == '#': + comment_token = line[pos:].rstrip('\r\n') + nl_pos = pos + len(comment_token) + yield (COMMENT, comment_token, (lnum, pos), + (lnum, pos + len(comment_token)), line) + yield (NL, line[nl_pos:], (lnum, nl_pos), (lnum, len(line)), line) + else: + yield ((NL, COMMENT)[line[pos] == '#'], line[pos:], (lnum, pos), + (lnum, len(line)), line) + continue + + if column > indents[-1]: # count indents or dedents + indents.append(column) + yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line) + while column < indents[-1]: + if column not in indents: + raise IndentationError( + 'unindent does not match any outer indentation level', + ('', lnum, pos, line)) + indents = indents[:-1] + + if async_def and async_def_indent >= indents[-1]: + async_def = False + async_def_nl = False + async_def_indent = 0 + + yield (DEDENT, '', (lnum, pos), (lnum, pos), line) + + if async_def and async_def_nl and async_def_indent >= indents[-1]: + async_def = False + async_def_nl = False + async_def_indent = 0 + + else: # continued statement + if not line: + raise TokenError('EOF in multi-line statement', (lnum, 0)) + continued = 0 + + while pos < max: + pseudomatch = pseudoprog.match(line, pos) + if pseudomatch: # scan for tokens + start, end = pseudomatch.span(1) + spos, epos, pos = (lnum, start), (lnum, end), end + token, initial = line[start:end], line[start] + + if initial in string.digits or \ + (initial == '.' and token != '.'): # ordinary number + yield (NUMBER, token, spos, epos, line) + elif initial in '\r\n': + newline = NEWLINE + if parenlev > 0: + newline = NL + elif async_def: + async_def_nl = True + if stashed: + yield stashed + stashed = None + yield (newline, token, spos, epos, line) + + elif initial == '#': + assert not token.endswith('\n') + if stashed: + yield stashed + stashed = None + yield (COMMENT, token, spos, epos, line) + elif token in triple_quoted: + endprog = endprogs[token] + endmatch = endprog.match(line, pos) + if endmatch: # all on one line + pos = endmatch.end(0) + token = line[start:pos] + if stashed: + yield stashed + stashed = None + yield (STRING, token, spos, (lnum, pos), line) + else: + strstart = (lnum, start) # multiple lines + contstr = line[start:] + contline = line + break + elif initial in single_quoted or \ + token[:2] in single_quoted or \ + token[:3] in single_quoted: + if token[-1] == '\n': # continued string + strstart = (lnum, start) # noqa: F841 + endprog = ( + endprogs[initial] or endprogs[token[1]] or endprogs[token[2]]) + contstr, needcont = line[start:], 1 + contline = line + break + else: # ordinary string + if stashed: + yield stashed + stashed = None + yield (STRING, token, spos, epos, line) + elif initial.isidentifier(): # ordinary name + if token in ('async', 'await'): + if async_def: + yield (ASYNC if token == 'async' else AWAIT, token, spos, epos, + line) + continue + + tok = (NAME, token, spos, epos, line) + if token == 'async' and not stashed: + stashed = tok + continue + + if token in ('def', 'for'): + if (stashed and stashed[0] == NAME and stashed[1] == 'async'): + + if token == 'def': + async_def = True + async_def_indent = indents[-1] + + yield (ASYNC, stashed[1], stashed[2], stashed[3], stashed[4]) + stashed = None + + if stashed: + yield stashed + stashed = None + + yield tok + elif initial == '\\': # continued stmt + # This yield is new; needed for better idempotency: + if stashed: + yield stashed + stashed = None + yield (NL, token, spos, (lnum, pos), line) + continued = 1 + else: + if initial in '([{': + parenlev = parenlev + 1 + elif initial in ')]}': + parenlev = parenlev - 1 + if stashed: + yield stashed + stashed = None + yield (OP, token, spos, epos, line) + else: + yield (ERRORTOKEN, line[pos], (lnum, pos), (lnum, pos + 1), line) + pos = pos + 1 + + if stashed: + yield stashed + stashed = None + + for indent in indents[1:]: # pop remaining indent levels + yield (DEDENT, '', (lnum, 0), (lnum, 0), '') + yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '') + + +if __name__ == '__main__': # testing + import sys + if len(sys.argv) > 1: + tokenize(open(sys.argv[1]).readline) + else: + tokenize(sys.stdin.readline) diff --git a/third_party/yapf_third_party/_ylib2to3/pygram.py b/third_party/yapf_third_party/_ylib2to3/pygram.py new file mode 100644 index 000000000..4267c3610 --- /dev/null +++ b/third_party/yapf_third_party/_ylib2to3/pygram.py @@ -0,0 +1,40 @@ +# Copyright 2006 Google, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. +"""Export the Python grammar and symbols.""" + +# Python imports +import os + +# Local imports +from .pgen2 import driver + +# The grammar file +_GRAMMAR_FILE = os.path.join(os.path.dirname(__file__), 'Grammar.txt') +_PATTERN_GRAMMAR_FILE = os.path.join( + os.path.dirname(__file__), 'PatternGrammar.txt') + + +class Symbols(object): + + def __init__(self, grammar): + """Initializer. + + Creates an attribute for each grammar symbol (nonterminal), + whose value is the symbol's type (an int >= 256). + """ + for name, symbol in grammar.symbol2number.items(): + setattr(self, name, symbol) + + +python_grammar = driver.load_grammar(_GRAMMAR_FILE) + +python_symbols = Symbols(python_grammar) + +python_grammar_no_print_statement = python_grammar.copy() +del python_grammar_no_print_statement.keywords['print'] + +python_grammar_no_print_and_exec_statement = python_grammar_no_print_statement.copy() # yapf: disable # noqa: E501 +del python_grammar_no_print_and_exec_statement.keywords['exec'] + +pattern_grammar = driver.load_grammar(_PATTERN_GRAMMAR_FILE) +pattern_symbols = Symbols(pattern_grammar) diff --git a/third_party/yapf_third_party/_ylib2to3/pytree.py b/third_party/yapf_third_party/_ylib2to3/pytree.py new file mode 100644 index 000000000..ea9767be9 --- /dev/null +++ b/third_party/yapf_third_party/_ylib2to3/pytree.py @@ -0,0 +1,861 @@ +# Copyright 2006 Google, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. +""" +Python parse tree definitions. + +This is a very concrete parse tree; we need to keep every token and +even the comments and whitespace between tokens. + +There's also a pattern matching implementation here. +""" + +__author__ = 'Guido van Rossum ' + +import sys +from io import StringIO +from typing import List +from typing import Optional +from typing import Text +from typing import Tuple +from typing import Union + +HUGE = 0x7FFFFFFF # maximum repeat count, default max + +_type_reprs = {} + + +def type_repr(type_num): + global _type_reprs + if not _type_reprs: + from .pygram import python_symbols + + # printing tokens is possible but not as useful + # from .pgen2 import token // token.__dict__.items(): + for name, val in python_symbols.__dict__.items(): + if isinstance(val, int): + _type_reprs[val] = name + return _type_reprs.setdefault(type_num, type_num) + + +NL = Union['Node', 'Leaf'] +Context = Tuple[Text, Tuple[int, int]] +RawNode = Tuple[int, Optional[Text], Optional[Context], Optional[List[NL]]] + + +class Base(object): + """ + Abstract base class for Node and Leaf. + + This provides some default functionality and boilerplate using the + template pattern. + + A node may be a subnode of at most one parent. + """ + + # Default values for instance variables + type = None # int: token number (< 256) or symbol number (>= 256) + parent = None # Parent node pointer, or None + children = () # Tuple of subnodes + was_changed = False + was_checked = False + + def __new__(cls, *args, **kwds): + """Constructor that prevents Base from being instantiated.""" + assert cls is not Base, 'Cannot instantiate Base' + return object.__new__(cls) + + def __eq__(self, other): + """ + Compare two nodes for equality. + + This calls the method _eq(). + """ + if self.__class__ is not other.__class__: + return NotImplemented + return self._eq(other) + + __hash__ = None # For Py3 compatibility. + + def _eq(self, other): + """ + Compare two nodes for equality. + + This is called by __eq__ and __ne__. It is only called if the two nodes + have the same type. This must be implemented by the concrete subclass. + Nodes should be considered equal if they have the same structure, + ignoring the prefix string and other context information. + """ + raise NotImplementedError + + def clone(self): + """ + Return a cloned (deep) copy of self. + + This must be implemented by the concrete subclass. + """ + raise NotImplementedError + + def post_order(self): + """ + Return a post-order iterator for the tree. + + This must be implemented by the concrete subclass. + """ + raise NotImplementedError + + def pre_order(self): + """ + Return a pre-order iterator for the tree. + + This must be implemented by the concrete subclass. + """ + raise NotImplementedError + + def replace(self, new): + """Replace this node with a new one in the parent.""" + assert self.parent is not None, str(self) + assert new is not None + if not isinstance(new, list): + new = [new] + l_children = [] + found = False + for ch in self.parent.children: + if ch is self: + assert not found, (self.parent.children, self, new) + if new is not None: + l_children.extend(new) + found = True + else: + l_children.append(ch) + assert found, (self.children, self, new) + self.parent.changed() + self.parent.children = l_children + for x in new: + x.parent = self.parent + self.parent = None + + def get_lineno(self): + """Return the line number which generated the invocant node.""" + node = self + while not isinstance(node, Leaf): + if not node.children: + return + node = node.children[0] + return node.lineno + + def changed(self): + if self.parent: + self.parent.changed() + self.was_changed = True + + def remove(self): + """ + Remove the node from the tree. Returns the position of the node in its + parent's children before it was removed. + """ + if self.parent: + for i, node in enumerate(self.parent.children): + if node is self: + self.parent.changed() + del self.parent.children[i] + self.parent = None + return i + + @property + def next_sibling(self): + """ + The node immediately following the invocant in their parent's children + list. If the invocant does not have a next sibling, it is None + """ + if self.parent is None: + return None + + # Can't use index(); we need to test by identity + for i, child in enumerate(self.parent.children): + if child is self: + try: + return self.parent.children[i + 1] + except IndexError: + return None + + @property + def prev_sibling(self): + """ + The node immediately preceding the invocant in their parent's children + list. If the invocant does not have a previous sibling, it is None. + """ + if self.parent is None: + return None + + # Can't use index(); we need to test by identity + for i, child in enumerate(self.parent.children): + if child is self: + if i == 0: + return None + return self.parent.children[i - 1] + + def leaves(self): + for child in self.children: + yield from child.leaves() + + def depth(self): + if self.parent is None: + return 0 + return 1 + self.parent.depth() + + def get_suffix(self): + """ + Return the string immediately following the invocant node. This is + effectively equivalent to node.next_sibling.prefix + """ + next_sib = self.next_sibling + if next_sib is None: + return '' + return next_sib.prefix + + if sys.version_info < (3, 0): + + def __str__(self): + return str(self).encode('ascii') + + +class Node(Base): + """Concrete implementation for interior nodes.""" + + def __init__(self, + type, + children, + context=None, + prefix=None, + fixers_applied=None): + """ + Initializer. + + Takes a type constant (a symbol number >= 256), a sequence of + child nodes, and an optional context keyword argument. + + As a side effect, the parent pointers of the children are updated. + """ + assert type >= 256, type + self.type = type + self.children = list(children) + for ch in self.children: + assert ch.parent is None, repr(ch) + ch.parent = self + if prefix is not None: + self.prefix = prefix + if fixers_applied: + self.fixers_applied = fixers_applied[:] + else: + self.fixers_applied = None + + def __repr__(self): + """Return a canonical string representation.""" + return '%s(%s, %r)' % (self.__class__.__name__, type_repr( + self.type), self.children) + + def __unicode__(self): + """ + Return a pretty string representation. + + This reproduces the input source exactly. + """ + return ''.join(map(str, self.children)) + + if sys.version_info > (3, 0): + __str__ = __unicode__ + + def _eq(self, other): + """Compare two nodes for equality.""" + return (self.type, self.children) == (other.type, other.children) + + def clone(self): + """Return a cloned (deep) copy of self.""" + return Node( + self.type, [ch.clone() for ch in self.children], + fixers_applied=self.fixers_applied) + + def post_order(self): + """Return a post-order iterator for the tree.""" + for child in self.children: + yield from child.post_order() + yield self + + def pre_order(self): + """Return a pre-order iterator for the tree.""" + yield self + for child in self.children: + yield from child.pre_order() + + @property + def prefix(self): + """ + The whitespace and comments preceding this node in the input. + """ + if not self.children: + return '' + return self.children[0].prefix + + @prefix.setter + def prefix(self, prefix): + if self.children: + self.children[0].prefix = prefix + + def set_child(self, i, child): + """ + Equivalent to 'node.children[i] = child'. This method also sets the + child's parent attribute appropriately. + """ + child.parent = self + self.children[i].parent = None + self.children[i] = child + self.changed() + + def insert_child(self, i, child): + """ + Equivalent to 'node.children.insert(i, child)'. This method also sets + the child's parent attribute appropriately. + """ + child.parent = self + self.children.insert(i, child) + self.changed() + + def append_child(self, child): + """ + Equivalent to 'node.children.append(child)'. This method also sets the + child's parent attribute appropriately. + """ + child.parent = self + self.children.append(child) + self.changed() + + +class Leaf(Base): + """Concrete implementation for leaf nodes.""" + + # Default values for instance variables + _prefix = '' # Whitespace and comments preceding this token in the input + lineno = 0 # Line where this token starts in the input + column = 0 # Column where this token tarts in the input + + def __init__(self, type, value, context=None, prefix=None, fixers_applied=[]): + """ + Initializer. + + Takes a type constant (a token number < 256), a string value, and an + optional context keyword argument. + """ + assert 0 <= type < 256, type + if context is not None: + self._prefix, (self.lineno, self.column) = context + self.type = type + self.value = value + if prefix is not None: + self._prefix = prefix + self.fixers_applied = fixers_applied[:] + + def __repr__(self): + """Return a canonical string representation.""" + return '%s(%r, %r)' % (self.__class__.__name__, self.type, self.value) + + def __unicode__(self): + """ + Return a pretty string representation. + + This reproduces the input source exactly. + """ + return self.prefix + str(self.value) + + if sys.version_info > (3, 0): + __str__ = __unicode__ + + def _eq(self, other): + """Compare two nodes for equality.""" + return (self.type, self.value) == (other.type, other.value) + + def clone(self): + """Return a cloned (deep) copy of self.""" + return Leaf( + self.type, + self.value, (self.prefix, (self.lineno, self.column)), + fixers_applied=self.fixers_applied) + + def leaves(self): + yield self + + def post_order(self): + """Return a post-order iterator for the tree.""" + yield self + + def pre_order(self): + """Return a pre-order iterator for the tree.""" + yield self + + @property + def prefix(self): + """ + The whitespace and comments preceding this token in the input. + """ + return self._prefix + + @prefix.setter + def prefix(self, prefix): + self.changed() + self._prefix = prefix + + +def convert(gr, raw_node): + """ + Convert raw node information to a Node or Leaf instance. + + This is passed to the parser driver which calls it whenever a reduction of a + grammar rule produces a new complete node, so that the tree is build + strictly bottom-up. + """ + type, value, context, children = raw_node + if children or type in gr.number2symbol: + # If there's exactly one child, return that child instead of + # creating a new node. + if len(children) == 1: + return children[0] + return Node(type, children, context=context) + else: + return Leaf(type, value, context=context) + + +class BasePattern(object): + """ + A pattern is a tree matching pattern. + + It looks for a specific node type (token or symbol), and + optionally for a specific content. + + This is an abstract base class. There are three concrete + subclasses: + + - LeafPattern matches a single leaf node; + - NodePattern matches a single node (usually non-leaf); + - WildcardPattern matches a sequence of nodes of variable length. + """ + + # Defaults for instance variables + type = None # Node type (token if < 256, symbol if >= 256) + content = None # Optional content matching pattern + name = None # Optional name used to store match in results dict + + def __new__(cls, *args, **kwds): + """Constructor that prevents BasePattern from being instantiated.""" + assert cls is not BasePattern, 'Cannot instantiate BasePattern' + return object.__new__(cls) + + def __repr__(self): + args = [type_repr(self.type), self.content, self.name] + while args and args[-1] is None: + del args[-1] + return '%s(%s)' % (self.__class__.__name__, ', '.join(map(repr, args))) + + def optimize(self): + """ + A subclass can define this as a hook for optimizations. + + Returns either self or another node with the same effect. + """ + return self + + def match(self, node, results=None): + """ + Does this pattern exactly match a node? + + Returns True if it matches, False if not. + + If results is not None, it must be a dict which will be + updated with the nodes matching named subpatterns. + + Default implementation for non-wildcard patterns. + """ + if self.type is not None and node.type != self.type: + return False + if self.content is not None: + r = None + if results is not None: + r = {} + if not self._submatch(node, r): + return False + if r: + results.update(r) + if results is not None and self.name: + results[self.name] = node + return True + + def match_seq(self, nodes, results=None): + """ + Does this pattern exactly match a sequence of nodes? + + Default implementation for non-wildcard patterns. + """ + if len(nodes) != 1: + return False + return self.match(nodes[0], results) + + def generate_matches(self, nodes): + """ + Generator yielding all matches for this pattern. + + Default implementation for non-wildcard patterns. + """ + r = {} + if nodes and self.match(nodes[0], r): + yield 1, r + + +class LeafPattern(BasePattern): + + def __init__(self, type=None, content=None, name=None): + """ + Initializer. Takes optional type, content, and name. + + The type, if given must be a token type (< 256). If not given, + this matches any *leaf* node; the content may still be required. + + The content, if given, must be a string. + + If a name is given, the matching node is stored in the results + dict under that key. + """ + if type is not None: + assert 0 <= type < 256, type + if content is not None: + assert isinstance(content, str), repr(content) + self.type = type + self.content = content + self.name = name + + def match(self, node, results=None): + """Override match() to insist on a leaf node.""" + if not isinstance(node, Leaf): + return False + return BasePattern.match(self, node, results) + + def _submatch(self, node, results=None): + """ + Match the pattern's content to the node's children. + + This assumes the node type matches and self.content is not None. + + Returns True if it matches, False if not. + + If results is not None, it must be a dict which will be + updated with the nodes matching named subpatterns. + + When returning False, the results dict may still be updated. + """ + return self.content == node.value + + +class NodePattern(BasePattern): + + wildcards = False + + def __init__(self, type=None, content=None, name=None): + """ + Initializer. Takes optional type, content, and name. + + The type, if given, must be a symbol type (>= 256). If the + type is None this matches *any* single node (leaf or not), + except if content is not None, in which it only matches + non-leaf nodes that also match the content pattern. + + The content, if not None, must be a sequence of Patterns that + must match the node's children exactly. If the content is + given, the type must not be None. + + If a name is given, the matching node is stored in the results + dict under that key. + """ + if type is not None: + assert type >= 256, type + if content is not None: + assert not isinstance(content, str), repr(content) + content = list(content) + for i, item in enumerate(content): + assert isinstance(item, BasePattern), (i, item) + if isinstance(item, WildcardPattern): + self.wildcards = True + self.type = type + self.content = content + self.name = name + + def _submatch(self, node, results=None): + """ + Match the pattern's content to the node's children. + + This assumes the node type matches and self.content is not None. + + Returns True if it matches, False if not. + + If results is not None, it must be a dict which will be + updated with the nodes matching named subpatterns. + + When returning False, the results dict may still be updated. + """ + if self.wildcards: + for c, r in generate_matches(self.content, node.children): + if c == len(node.children): + if results is not None: + results.update(r) + return True + return False + if len(self.content) != len(node.children): + return False + for subpattern, child in zip(self.content, node.children): + if not subpattern.match(child, results): + return False + return True + + +class WildcardPattern(BasePattern): + """ + A wildcard pattern can match zero or more nodes. + + This has all the flexibility needed to implement patterns like: + + .* .+ .? .{m,n} + (a b c | d e | f) + (...)* (...)+ (...)? (...){m,n} + + except it always uses non-greedy matching. + """ + + def __init__(self, content=None, min=0, max=HUGE, name=None): + """ + Initializer. + + Args: + content: optional sequence of subsequences of patterns; + if absent, matches one node; + if present, each subsequence is an alternative [*] + min: optional minimum number of times to match, default 0 + max: optional maximum number of times to match, default HUGE + name: optional name assigned to this match + + [*] Thus, if content is [[a, b, c], [d, e], [f, g, h]] this is + equivalent to (a b c | d e | f g h); if content is None, + this is equivalent to '.' in regular expression terms. + The min and max parameters work as follows: + min=0, max=maxint: .* + min=1, max=maxint: .+ + min=0, max=1: .? + min=1, max=1: . + If content is not None, replace the dot with the parenthesized + list of alternatives, e.g. (a b c | d e | f g h)* + """ + assert 0 <= min <= max <= HUGE, (min, max) + if content is not None: + content = tuple(map(tuple, content)) # Protect against alterations + # Check sanity of alternatives + assert len(content), repr(content) # Can't have zero alternatives + for alt in content: + assert len(alt), repr(alt) # Can have empty alternatives + self.content = content + self.min = min + self.max = max + self.name = name + + def optimize(self): + """Optimize certain stacked wildcard patterns.""" + subpattern = None + if (self.content is not None and len(self.content) == 1 and + len(self.content[0]) == 1): + subpattern = self.content[0][0] + if self.min == 1 and self.max == 1: + if self.content is None: + return NodePattern(name=self.name) + if subpattern is not None and self.name == subpattern.name: + return subpattern.optimize() + if (self.min <= 1 and isinstance(subpattern, WildcardPattern) and + subpattern.min <= 1 and self.name == subpattern.name): + return WildcardPattern(subpattern.content, self.min * subpattern.min, + self.max * subpattern.max, subpattern.name) + return self + + def match(self, node, results=None): + """Does this pattern exactly match a node?""" + return self.match_seq([node], results) + + def match_seq(self, nodes, results=None): + """Does this pattern exactly match a sequence of nodes?""" + for c, r in self.generate_matches(nodes): + if c == len(nodes): + if results is not None: + results.update(r) + if self.name: + results[self.name] = list(nodes) + return True + return False + + def generate_matches(self, nodes): + """ + Generator yielding matches for a sequence of nodes. + + Args: + nodes: sequence of nodes + + Yields: + (count, results) tuples where: + count: the match comprises nodes[:count]; + results: dict containing named submatches. + """ + if self.content is None: + # Shortcut for special case (see __init__.__doc__) + for count in range(self.min, 1 + min(len(nodes), self.max)): + r = {} + if self.name: + r[self.name] = nodes[:count] + yield count, r + elif self.name == 'bare_name': + yield self._bare_name_matches(nodes) + else: + # The reason for this is that hitting the recursion limit usually + # results in some ugly messages about how RuntimeErrors are being + # ignored. We only have to do this on CPython, though, because other + # implementations don't have this nasty bug in the first place. + if hasattr(sys, 'getrefcount'): + save_stderr = sys.stderr + sys.stderr = StringIO() + try: + for count, r in self._recursive_matches(nodes, 0): + if self.name: + r[self.name] = nodes[:count] + yield count, r + except RuntimeError: + # We fall back to the iterative pattern matching scheme if the recursive + # scheme hits the recursion limit. + for count, r in self._iterative_matches(nodes): + if self.name: + r[self.name] = nodes[:count] + yield count, r + finally: + if hasattr(sys, 'getrefcount'): + sys.stderr = save_stderr + + def _iterative_matches(self, nodes): + """Helper to iteratively yield the matches.""" + nodelen = len(nodes) + if 0 >= self.min: + yield 0, {} + + results = [] + # generate matches that use just one alt from self.content + for alt in self.content: + for c, r in generate_matches(alt, nodes): + yield c, r + results.append((c, r)) + + # for each match, iterate down the nodes + while results: + new_results = [] + for c0, r0 in results: + # stop if the entire set of nodes has been matched + if c0 < nodelen and c0 <= self.max: + for alt in self.content: + for c1, r1 in generate_matches(alt, nodes[c0:]): + if c1 > 0: + r = {} + r.update(r0) + r.update(r1) + yield c0 + c1, r + new_results.append((c0 + c1, r)) + results = new_results + + def _bare_name_matches(self, nodes): + """Special optimized matcher for bare_name.""" + count = 0 + r = {} + done = False + max = len(nodes) + while not done and count < max: + done = True + for leaf in self.content: + if leaf[0].match(nodes[count], r): + count += 1 + done = False + break + r[self.name] = nodes[:count] + return count, r + + def _recursive_matches(self, nodes, count): + """Helper to recursively yield the matches.""" + assert self.content is not None + if count >= self.min: + yield 0, {} + if count < self.max: + for alt in self.content: + for c0, r0 in generate_matches(alt, nodes): + for c1, r1 in self._recursive_matches(nodes[c0:], count + 1): + r = {} + r.update(r0) + r.update(r1) + yield c0 + c1, r + + +class NegatedPattern(BasePattern): + + def __init__(self, content=None): + """ + Initializer. + + The argument is either a pattern or None. If it is None, this + only matches an empty sequence (effectively '$' in regex + lingo). If it is not None, this matches whenever the argument + pattern doesn't have any matches. + """ + if content is not None: + assert isinstance(content, BasePattern), repr(content) + self.content = content + + def match(self, node): + # We never match a node in its entirety + return False + + def match_seq(self, nodes): + # We only match an empty sequence of nodes in its entirety + return len(nodes) == 0 + + def generate_matches(self, nodes): + if self.content is None: + # Return a match if there is an empty sequence + if len(nodes) == 0: + yield 0, {} + else: + # Return a match if the argument pattern has no matches + for c, r in self.content.generate_matches(nodes): + return + yield 0, {} + + +def generate_matches(patterns, nodes): + """ + Generator yielding matches for a sequence of patterns and nodes. + + Args: + patterns: a sequence of patterns + nodes: a sequence of nodes + + Yields: + (count, results) tuples where: + count: the entire sequence of patterns matches nodes[:count]; + results: dict containing named submatches. + """ + if not patterns: + yield 0, {} + else: + p, rest = patterns[0], patterns[1:] + for c0, r0 in p.generate_matches(nodes): + if not rest: + yield c0, r0 + else: + for c1, r1 in generate_matches(rest, nodes[c0:]): + r = {} + r.update(r0) + r.update(r1) + yield c0 + c1, r diff --git a/third_party/yapf_third_party/yapf_diff/LICENSE b/third_party/yapf_third_party/yapf_diff/LICENSE new file mode 100644 index 000000000..bd8b243df --- /dev/null +++ b/third_party/yapf_third_party/yapf_diff/LICENSE @@ -0,0 +1,218 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +--- LLVM Exceptions to the Apache 2.0 License ---- + +As an exception, if, as a result of your compiling your source code, portions +of this Software are embedded into an Object form of such source code, you +may redistribute such embedded portions in such Object form without complying +with the conditions of Sections 4(a), 4(b) and 4(d) of the License. + +In addition, if you combine or link compiled forms of this Software with +software that is licensed under the GPLv2 ("Combined Software") and if a +court of competent jurisdiction determines that the patent provision (Section +3), the indemnity provision (Section 9) or other Section of the License +conflicts with the conditions of the GPLv2, you may retroactively and +prospectively choose to deem waived or otherwise exclude such Section(s) of +the License, but only in their entirety and only with respect to the Combined +Software. diff --git a/third_party/yapf_third_party/yapf_diff/__init__.py b/third_party/yapf_third_party/yapf_diff/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/third_party/yapf_third_party/yapf_diff/yapf_diff.py b/third_party/yapf_third_party/yapf_diff/yapf_diff.py new file mode 100644 index 000000000..a22abd953 --- /dev/null +++ b/third_party/yapf_third_party/yapf_diff/yapf_diff.py @@ -0,0 +1,140 @@ +# Modified copy of clang-format-diff.py that works with yapf. +# +# Licensed under the Apache License, Version 2.0 (the "License") with LLVM +# Exceptions; you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://llvm.org/LICENSE.txt +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This script reads input from a unified diff and reformats all the changed +lines. This is useful to reformat all the lines touched by a specific patch. +Example usage for git/svn users: + + git diff -U0 --no-color --relative HEAD^ | yapf-diff -i + svn diff --diff-cmd=diff -x-U0 | yapf-diff -p0 -i + +It should be noted that the filename contained in the diff is used unmodified +to determine the source file to update. Users calling this script directly +should be careful to ensure that the path in the diff is correct relative to the +current working directory. +""" + +import argparse +import difflib +import re +import subprocess +import sys +from io import StringIO + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '-i', + '--in-place', + action='store_true', + default=False, + help='apply edits to files instead of displaying a diff') + parser.add_argument( + '-p', + '--prefix', + metavar='NUM', + default=1, + help='strip the smallest prefix containing P slashes') + parser.add_argument( + '--regex', + metavar='PATTERN', + default=None, + help='custom pattern selecting file paths to reformat ' + '(case sensitive, overrides -iregex)') + parser.add_argument( + '--iregex', + metavar='PATTERN', + default=r'.*\.(py)', + help='custom pattern selecting file paths to reformat ' + '(case insensitive, overridden by -regex)') + parser.add_argument( + '-v', + '--verbose', + action='store_true', + help='be more verbose, ineffective without -i') + parser.add_argument( + '--style', + help='specify formatting style: either a style name (for ' + 'example "pep8" or "google"), or the name of a file with ' + 'style settings. The default is pep8 unless a ' + '.style.yapf or setup.cfg file located in one of the ' + 'parent directories of the source file (or current ' + 'directory for stdin)') + parser.add_argument( + '--binary', default='yapf', help='location of binary to use for yapf') + args = parser.parse_args() + + # Extract changed lines for each file. + filename = None + lines_by_file = {} + for line in sys.stdin: + match = re.search(r'^\+\+\+\ (.*?/){%s}(\S*)' % args.prefix, line) + if match: + filename = match.group(2) + if filename is None: + continue + + if args.regex is not None: + if not re.match('^%s$' % args.regex, filename): + continue + elif not re.match('^%s$' % args.iregex, filename, re.IGNORECASE): + continue + + match = re.search(r'^@@.*\+(\d+)(,(\d+))?', line) + if match: + start_line = int(match.group(1)) + line_count = 1 + if match.group(3): + line_count = int(match.group(3)) + if line_count == 0: + continue + end_line = start_line + line_count - 1 + lines_by_file.setdefault(filename, []).extend( + ['--lines', str(start_line) + '-' + str(end_line)]) + + # Reformat files containing changes in place. + for filename, lines in lines_by_file.items(): + if args.in_place and args.verbose: + print('Formatting {}'.format(filename)) + command = [args.binary, filename] + if args.in_place: + command.append('-i') + command.extend(lines) + if args.style: + command.extend(['--style', args.style]) + p = subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=None, + stdin=subprocess.PIPE, + universal_newlines=True) + stdout, stderr = p.communicate() + if p.returncode != 0: + sys.exit(p.returncode) + + if not args.in_place: + with open(filename) as f: + code = f.readlines() + formatted_code = StringIO(stdout).readlines() + diff = difflib.unified_diff(code, formatted_code, filename, filename, + '(before formatting)', '(after formatting)') + diff_string = ''.join(diff) + if len(diff_string) > 0: + sys.stdout.write(diff_string) + + +if __name__ == '__main__': + main() diff --git a/tox.ini b/tox.ini index 3109915bb..5e5b11e79 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,19 @@ [tox] -envlist=py27,py34,py35 +requires = + tox<4 + tox-pyenv + tox-wheel +envlist = py37,py38,py39,py310,py311,py312 +# tox-wheel alias for `wheel_pep517 = true` +isolated_build = True +distshare = ./dist [testenv] -commands= - python setup.py test +wheel = True +wheel_build_env = bdist_wheel +commands = python -m unittest discover -p '*_test.py' yapftests/ + +[testenv:bdist_wheel] + +[testenv:sdist] +wheel = False diff --git a/yapf/__init__.py b/yapf/__init__.py index dfbb8ee2d..cf4be9379 100644 --- a/yapf/__init__.py +++ b/yapf/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2015-2016 Google Inc. All Rights Reserved. +# Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,26 +19,39 @@ that line. It then uses a priority queue to figure out what the best formatting is --- i.e., the formatting with the least penalty. -It differs from tools like autopep8 and pep8ify in that it doesn't just look for +It differs from tools like autopep8 in that it doesn't just look for violations of the style guide, but looks at the module as a whole, making formatting decisions based on what's the best format for each line. If no filenames are specified, YAPF reads the code from stdin. """ -from __future__ import print_function import argparse +import codecs +import io import logging import os import sys +from yapf._version import __version__ from yapf.yapflib import errors from yapf.yapflib import file_resources -from yapf.yapflib import py3compat from yapf.yapflib import style from yapf.yapflib import yapf_api -__version__ = '0.11.0' + +def _raw_input(): + wrapper = io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8') + return wrapper.buffer.raw.readall().decode('utf-8') + + +def _removeBOM(source): + """Remove any Byte-order-Mark bytes from the beginning of a file.""" + bom = codecs.BOM_UTF8 + bom = bom.decode('utf-8') + if source.startswith(bom): + return source[len(bom):] + return source def main(argv): @@ -49,86 +62,18 @@ def main(argv): in argv[0]). Returns: - 0 if there were no changes, non-zero otherwise. + Zero on successful program termination, non-zero otherwise. + With --diff: zero if there were no changes, non-zero otherwise. Raises: YapfError: if none of the supplied files were Python files. """ - parser = argparse.ArgumentParser(description='Formatter for Python code.') - parser.add_argument( - '-v', - '--version', - action='store_true', - help='show version number and exit') - - diff_inplace_group = parser.add_mutually_exclusive_group() - diff_inplace_group.add_argument( - '-d', - '--diff', - action='store_true', - help='print the diff for the fixed source') - diff_inplace_group.add_argument( - '-i', - '--in-place', - action='store_true', - help='make changes to files in place') - - lines_recursive_group = parser.add_mutually_exclusive_group() - lines_recursive_group.add_argument( - '-r', - '--recursive', - action='store_true', - help='run recursively over directories') - lines_recursive_group.add_argument( - '-l', - '--lines', - metavar='START-END', - action='append', - default=None, - help='range of lines to reformat, one-based') - - parser.add_argument( - '-e', - '--exclude', - metavar='PATTERN', - action='append', - default=None, - help='patterns for files to exclude from formatting') - parser.add_argument( - '--style', - action='store', - help=('specify formatting style: either a style name (for example "pep8" ' - 'or "google"), or the name of a file with style settings. The ' - 'default is pep8 unless a %s or %s file located in one of the ' - 'parent directories of the source file (or current directory for ' - 'stdin)' % (style.LOCAL_STYLE, style.SETUP_CONFIG))) - parser.add_argument( - '--style-help', - action='store_true', - help=('show style settings and exit; this output can be ' - 'saved to .style.yapf to make your settings ' - 'permanent')) - parser.add_argument( - '--no-local-style', - action='store_true', - help="don't search for local style definition") - parser.add_argument('--verify', action='store_true', help=argparse.SUPPRESS) - - parser.add_argument('files', nargs='*') + parser = _BuildParser() args = parser.parse_args(argv[1:]) - - if args.version: - print('yapf {}'.format(__version__)) - return 0 + style_config = args.style if args.style_help: - style.SetGlobalStyle(style.CreateStyleFromConfig(args.style)) - print('[style]') - for option, docstring in sorted(style.Help().items()): - for line in docstring.splitlines(): - print('#', line) - print(option, '=', style.Get(option), sep='') - print() + _PrintHelp(args) return 0 if args.lines and len(args.files) > 1: @@ -143,30 +88,51 @@ def main(argv): original_source = [] while True: + # Test that sys.stdin has the "closed" attribute. When using pytest, it + # co-opts sys.stdin, which makes the "main_tests.py" fail. This is gross. + if hasattr(sys.stdin, 'closed') and sys.stdin.closed: + break try: # Use 'raw_input' instead of 'sys.stdin.read', because otherwise the # user will need to hit 'Ctrl-D' more than once if they're inputting # the program by hand. 'raw_input' throws an EOFError exception if # 'Ctrl-D' is pressed, which makes it easy to bail out of this loop. - original_source.append(py3compat.raw_input()) + original_source.append(_raw_input()) except EOFError: break - style_config = args.style + except KeyboardInterrupt: + return 1 + if style_config is None and not args.no_local_style: style_config = file_resources.GetDefaultStyleForDir(os.getcwd()) - reformatted_source, changed = yapf_api.FormatCode( - py3compat.unicode('\n'.join(original_source) + '\n'), - filename='', - style_config=style_config, - lines=lines, - verify=args.verify) - sys.stdout.write(reformatted_source) - return 2 if changed else 0 + + source = [line.rstrip() for line in original_source] + source[0] = _removeBOM(source[0]) + + try: + reformatted_source, _ = yapf_api.FormatCode( + str('\n'.join(source).replace('\r\n', '\n') + '\n'), + filename='', + style_config=style_config, + lines=lines) + except errors.YapfError: + raise + except Exception as e: + raise errors.YapfError(errors.FormatErrorMsg(e)) + + file_resources.WriteReformattedCode('', reformatted_source) + return 0 + + # Get additional exclude patterns from ignorefile + exclude_patterns_from_ignore_file = file_resources.GetExcludePatternsForDir( + os.getcwd()) files = file_resources.GetCommandLineFiles(args.files, args.recursive, - args.exclude) + (args.exclude or []) + + exclude_patterns_from_ignore_file) if not files: - raise errors.YapfError('Input filenames did not match any python files') + raise errors.YapfError('input filenames did not match any python files') + changed = FormatFiles( files, lines, @@ -174,8 +140,28 @@ def main(argv): no_local_style=args.no_local_style, in_place=args.in_place, print_diff=args.diff, - verify=args.verify) - return 2 if changed else 0 + parallel=args.parallel, + quiet=args.quiet, + verbose=args.verbose, + print_modified=args.print_modified) + return 1 if changed and (args.diff or args.quiet) else 0 + + +def _PrintHelp(args): + """Prints the help menu.""" + + if args.style is None and not args.no_local_style: + args.style = file_resources.GetDefaultStyleForDir(os.getcwd()) + style.SetGlobalStyle(style.CreateStyleFromConfig(args.style)) + print('[style]') + for option, docstring in sorted(style.Help().items()): + for line in docstring.splitlines(): + print('#', line and ' ' or '', line, sep='') + option_value = style.Get(option) + if isinstance(option_value, (set, list)): + option_value = ', '.join(map(str, option_value)) + print(option.lower(), '=', option_value, sep='') + print() def FormatFiles(filenames, @@ -184,7 +170,10 @@ def FormatFiles(filenames, no_local_style=False, in_place=False, print_diff=False, - verify=True): + parallel=False, + quiet=False, + verbose=False, + print_modified=False): """Format a list of files. Arguments: @@ -199,36 +188,73 @@ def FormatFiles(filenames, in_place: (bool) Modify the files in place. print_diff: (bool) Instead of returning the reformatted source, return a diff that turns the formatted source into reformatter source. - verify: (bool) True if reformatted code should be verified for syntax. + parallel: (bool) True if should format multiple files in parallel. + quiet: (bool) True if should output nothing. + verbose: (bool) True if should print out filenames while processing. + print_modified: (bool) True if should print out filenames of modified files. Returns: True if the source code changed in any of the files being formatted. """ changed = False - for filename in filenames: - logging.info('Reformatting %s', filename) - if style_config is None and not no_local_style: - style_config = ( - file_resources.GetDefaultStyleForDir(os.path.dirname(filename))) - try: - reformatted_code, encoding, has_change = yapf_api.FormatFile( - filename, - in_place=in_place, - style_config=style_config, - lines=lines, - print_diff=print_diff, - verify=verify, - logger=logging.warning) - if has_change and reformatted_code is not None: - file_resources.WriteReformattedCode(filename, reformatted_code, - in_place, encoding) - changed |= has_change - except SyntaxError as e: - e.filename = filename - raise + if parallel: + import concurrent.futures # pylint: disable=g-import-not-at-top + import multiprocessing # pylint: disable=g-import-not-at-top + workers = min(multiprocessing.cpu_count(), len(filenames)) + with concurrent.futures.ProcessPoolExecutor(workers) as executor: + future_formats = [ + executor.submit(_FormatFile, filename, lines, style_config, + no_local_style, in_place, print_diff, quiet, verbose, + print_modified) for filename in filenames + ] + for future in concurrent.futures.as_completed(future_formats): + changed |= future.result() + else: + for filename in filenames: + changed |= _FormatFile(filename, lines, style_config, no_local_style, + in_place, print_diff, quiet, verbose, + print_modified) return changed +def _FormatFile(filename, + lines, + style_config=None, + no_local_style=False, + in_place=False, + print_diff=False, + quiet=False, + verbose=False, + print_modified=False): + """Format an individual file.""" + if verbose and not quiet: + print(f'Reformatting {filename}') + + if style_config is None and not no_local_style: + style_config = file_resources.GetDefaultStyleForDir( + os.path.dirname(filename)) + + try: + reformatted_code, encoding, has_change = yapf_api.FormatFile( + filename, + in_place=in_place, + style_config=style_config, + lines=lines, + print_diff=print_diff, + logger=logging.warning) + except errors.YapfError: + raise + except Exception as e: + raise errors.YapfError(errors.FormatErrorMsg(e)) + + if not in_place and not quiet and reformatted_code: + file_resources.WriteReformattedCode(filename, reformatted_code, encoding, + in_place) + if print_modified and has_change and in_place and not quiet: + print(f'Formatted {filename}') + return has_change + + def _GetLines(line_strings): """Parses the start and end lines from a line string like 'start-end'. @@ -249,11 +275,103 @@ def _GetLines(line_strings): if line[0] < 1: raise errors.YapfError('invalid start of line range: %r' % line) if line[0] > line[1]: - raise errors.YapfError('end comes before start in line range: %r', line) + raise errors.YapfError('end comes before start in line range: %r' % line) lines.append(tuple(line)) return lines +def _BuildParser(): + """Constructs the parser for the command line arguments. + + Returns: + An ArgumentParser instance for the CLI. + """ + parser = argparse.ArgumentParser( + prog='yapf', description='Formatter for Python code.') + parser.add_argument( + '-v', + '--version', + action='version', + version='%(prog)s {}'.format(__version__)) + + diff_inplace_quiet_group = parser.add_mutually_exclusive_group() + diff_inplace_quiet_group.add_argument( + '-d', + '--diff', + action='store_true', + help='print the diff for the fixed source') + diff_inplace_quiet_group.add_argument( + '-i', + '--in-place', + action='store_true', + help='make changes to files in place') + diff_inplace_quiet_group.add_argument( + '-q', + '--quiet', + action='store_true', + help='output nothing and set return value') + + lines_recursive_group = parser.add_mutually_exclusive_group() + lines_recursive_group.add_argument( + '-r', + '--recursive', + action='store_true', + help='run recursively over directories') + lines_recursive_group.add_argument( + '-l', + '--lines', + metavar='START-END', + action='append', + default=None, + help='range of lines to reformat, one-based') + + parser.add_argument( + '-e', + '--exclude', + metavar='PATTERN', + action='append', + default=None, + help='patterns for files to exclude from formatting') + parser.add_argument( + '--style', + action='store', + help=('specify formatting style: either a style name (for example "pep8" ' + 'or "google"), or the name of a file with style settings. The ' + 'default is pep8 unless a %s or %s or %s file located in the same ' + 'directory as the source or one of its parent directories ' + '(for stdin, the current directory is used).' % + (style.LOCAL_STYLE, style.SETUP_CONFIG, style.PYPROJECT_TOML))) + parser.add_argument( + '--style-help', + action='store_true', + help=('show style settings and exit; this output can be ' + 'saved to .style.yapf to make your settings ' + 'permanent')) + parser.add_argument( + '--no-local-style', + action='store_true', + help="don't search for local style definition") + parser.add_argument( + '-p', + '--parallel', + action='store_true', + help=('run YAPF in parallel when formatting multiple files.')) + parser.add_argument( + '-m', + '--print-modified', + action='store_true', + help='print out file names of modified files') + parser.add_argument( + '-vv', + '--verbose', + action='store_true', + help='print out file names while processing') + + parser.add_argument( + 'files', nargs='*', help='reads from stdin when no files are specified.') + return parser + + def run_main(): # pylint: disable=invalid-name try: sys.exit(main(sys.argv)) diff --git a/yapf/__main__.py b/yapf/__main__.py index b9c20bdd9..f1d0e322d 100644 --- a/yapf/__main__.py +++ b/yapf/__main__.py @@ -1,4 +1,4 @@ -# Copyright 2015-2016 Google Inc. All Rights Reserved. +# Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +"""Main entry point.""" +# pylint: disable=invalid-name import yapf yapf.run_main() diff --git a/yapf/_version.py b/yapf/_version.py new file mode 100644 index 000000000..1e79165d5 --- /dev/null +++ b/yapf/_version.py @@ -0,0 +1 @@ +__version__ = '0.43.0' diff --git a/yapf/pyparser/__init__.py b/yapf/pyparser/__init__.py new file mode 100644 index 000000000..1b6cc8061 --- /dev/null +++ b/yapf/pyparser/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/yapf/pyparser/pyparser.py b/yapf/pyparser/pyparser.py new file mode 100644 index 000000000..a7b4e33c3 --- /dev/null +++ b/yapf/pyparser/pyparser.py @@ -0,0 +1,163 @@ +# Copyright 2022 Bill Wendling, All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Simple Python Parser + +Parse Python code into a list of logical lines, represented by LogicalLine +objects. This uses Python's tokenizer to generate the tokens. As such, YAPF must +be run with the appropriate Python version---Python >=3.7 for Python 3.7 code, +Python >=3.8 for Python 3.8 code, etc. + +This parser uses Python's native "tokenizer" module to generate a list of tokens +for the source code. It then uses Python's native "ast" module to assign +subtypes, calculate split penalties, etc. + +A "logical line" produced by Python's "tokenizer" module ends with a +tokenize.NEWLINE, rather than a tokenize.NL, making it easy to separate them +out. Comments all end with a tokentizer.NL, so we need to make sure we don't +errantly pick up non-comment tokens when parsing comment blocks. + + ParseCode(): parse the code producing a list of logical lines. +""" + +# TODO: Call from yapf_api.FormatCode. + +import ast +import codecs +import os +import token +import tokenize +from io import StringIO +from tokenize import TokenInfo + +from yapf.pyparser import split_penalty_visitor +from yapf.yapflib import format_token +from yapf.yapflib import logical_line + +CONTINUATION = token.N_TOKENS + + +def ParseCode(unformatted_source, filename=''): + """Parse a string of Python code into logical lines. + + This provides an alternative entry point to YAPF. + + Arguments: + unformatted_source: (unicode) The code to format. + filename: (unicode) The name of the file being reformatted. + + Returns: + A list of LogicalLines. + + Raises: + An exception is raised if there's an error during AST parsing. + """ + if not unformatted_source.endswith(os.linesep): + unformatted_source += os.linesep + + try: + ast_tree = ast.parse(unformatted_source, filename) + ast.fix_missing_locations(ast_tree) + readline = StringIO(unformatted_source).readline + tokens = tokenize.generate_tokens(readline) + except Exception: + raise + + logical_lines = _CreateLogicalLines(tokens) + + # Process the logical lines. + split_penalty_visitor.SplitPenalty(logical_lines).visit(ast_tree) + + return logical_lines + + +def _CreateLogicalLines(tokens): + """Separate tokens into logical lines. + + Arguments: + tokens: (list of tokenizer.TokenInfo) Tokens generated by tokenizer. + + Returns: + A list of LogicalLines. + """ + formatted_tokens = [] + + # Convert tokens into "TokenInfo" and add tokens for continuation markers. + prev_tok = None + for tok in tokens: + tok = TokenInfo(*tok) + + if (prev_tok and prev_tok.line.rstrip().endswith('\\') and + prev_tok.start[0] < tok.start[0]): + ctok = TokenInfo( + type=CONTINUATION, + string='\\', + start=(prev_tok.start[0], prev_tok.start[1] + 1), + end=(prev_tok.end[0], prev_tok.end[0] + 2), + line=prev_tok.line) + ctok.lineno = ctok.start[0] + ctok.column = ctok.start[1] + ctok.value = '\\' + formatted_tokens.append(format_token.FormatToken(ctok, 'CONTINUATION')) + + tok.lineno = tok.start[0] + tok.column = tok.start[1] + tok.value = tok.string + formatted_tokens.append( + format_token.FormatToken(tok, token.tok_name[tok.type])) + prev_tok = tok + + # Generate logical lines. + logical_lines, cur_logical_line = [], [] + depth = 0 + for tok in formatted_tokens: + if tok.type == tokenize.ENDMARKER: + break + + if tok.type == tokenize.NEWLINE: + # End of a logical line. + logical_lines.append(logical_line.LogicalLine(depth, cur_logical_line)) + cur_logical_line = [] + elif tok.type == tokenize.INDENT: + depth += 1 + elif tok.type == tokenize.DEDENT: + depth -= 1 + elif tok.type == tokenize.NL: + pass + else: + if (cur_logical_line and not tok.type == tokenize.COMMENT and + cur_logical_line[0].type == tokenize.COMMENT): + # We were parsing a comment block, but now we have real code to worry + # about. Store the comment and carry on. + logical_lines.append(logical_line.LogicalLine(depth, cur_logical_line)) + cur_logical_line = [] + + cur_logical_line.append(tok) + + # Link the FormatTokens in each line together to form a doubly linked list. + for line in logical_lines: + previous = line.first + bracket_stack = [previous] if previous.OpensScope() else [] + for tok in line.tokens[1:]: + tok.previous_token = previous + previous.next_token = tok + previous = tok + + # Set up the "matching_bracket" attribute. + if tok.OpensScope(): + bracket_stack.append(tok) + elif tok.ClosesScope(): + bracket_stack[-1].matching_bracket = tok + tok.matching_bracket = bracket_stack.pop() + + return logical_lines diff --git a/yapf/pyparser/pyparser_utils.py b/yapf/pyparser/pyparser_utils.py new file mode 100644 index 000000000..dee2449d0 --- /dev/null +++ b/yapf/pyparser/pyparser_utils.py @@ -0,0 +1,103 @@ +# Copyright 2022 Bill Wendling, All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyParser-related utilities. + +This module collects various utilities related to the parse trees produced by +the pyparser. + + GetLogicalLine: produces a list of tokens from the logical lines within a + range. + GetTokensInSubRange: produces a sublist of tokens from a current token list + within a range. + GetTokenIndex: Get the index of a token. + GetNextTokenIndex: Get the index of the next token after a given position. + GetPrevTokenIndex: Get the index of the previous token before a given + position. + TokenStart: Convenience function to return the token's start as a tuple. + TokenEnd: Convenience function to return the token's end as a tuple. +""" + + +def GetLogicalLine(logical_lines, node): + """Get a list of tokens within the node's range from the logical lines.""" + start = TokenStart(node) + end = TokenEnd(node) + tokens = [] + + for line in logical_lines: + if line.start > end: + break + if line.start <= start or line.end >= end: + tokens.extend(GetTokensInSubRange(line.tokens, node)) + + return tokens + + +def GetTokensInSubRange(tokens, node): + """Get a subset of tokens representing the node.""" + start = TokenStart(node) + end = TokenEnd(node) + tokens_in_range = [] + + for tok in tokens: + tok_range = (tok.lineno, tok.column) + if tok_range >= start and tok_range < end: + tokens_in_range.append(tok) + + return tokens_in_range + + +def GetTokenIndex(tokens, pos): + """Get the index of the token at pos.""" + for index, token in enumerate(tokens): + if (token.lineno, token.column) == pos: + return index + + return None + + +def GetNextTokenIndex(tokens, pos): + """Get the index of the next token after pos.""" + for index, token in enumerate(tokens): + if (token.lineno, token.column) >= pos: + return index + + return None + + +def GetPrevTokenIndex(tokens, pos): + """Get the index of the previous token before pos.""" + for index, token in enumerate(tokens): + if index > 0 and (token.lineno, token.column) >= pos: + return index - 1 + + return None + + +def TokenStart(node): + return (node.lineno, node.col_offset) + + +def TokenEnd(node): + return (node.end_lineno, node.end_col_offset) + + +############################################################################# +# Code for debugging # +############################################################################# + + +def AstDump(node): + import ast + print(ast.dump(node, include_attributes=True, indent=4)) diff --git a/yapf/pyparser/pyparser_visitor.py.tmpl b/yapf/pyparser/pyparser_visitor.py.tmpl new file mode 100644 index 000000000..0d8a75e23 --- /dev/null +++ b/yapf/pyparser/pyparser_visitor.py.tmpl @@ -0,0 +1,646 @@ +# Copyright 2022 Bill Wendling, All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""AST visitor template. + +This is a template for a pyparser visitor. Example use: + + import ast + from io import StringIO + + from yapf.pyparser import pyparser_visitor + + def parse_code(source, filename): + ast_tree = ast.parse(source, filename) + readline = StringIO(source).readline + tokens = tokenize.generate_tokens(readline) + logical_lines = _CreateLogicalLines(tokens) + + pyparser_visitor.Visitor(logical_lines).visit(ast_tree) +""" + +import ast + + +# This is a skeleton of an AST visitor. +class Visitor(ast.NodeVisitor): + """Compute split penalties between tokens.""" + + def __init__(self, logical_lines): + super(Visitor, self).__init__() + self.logical_lines = logical_lines + + ############################################################################ + # Statements # + ############################################################################ + + def visit_FunctionDef(self, node): + # FunctionDef(name=Name, + # args=arguments( + # posonlyargs=[], + # args=[], + # vararg=[], + # kwonlyargs=[], + # kw_defaults=[], + # defaults=[]), + # body=[...], + # decorator_list=[Expr_1, Expr_2, ..., Expr_n], + # keywords=[]) + return self.generic_visit(node) + + def visit_AsyncFunctionDef(self, node): + # AsyncFunctionDef(name=Name, + # args=arguments( + # posonlyargs=[], + # args=[], + # vararg=[], + # kwonlyargs=[], + # kw_defaults=[], + # defaults=[]), + # body=[...], + # decorator_list=[Expr_1, Expr_2, ..., Expr_n], + # keywords=[]) + return self.generic_visit(node) + + def visit_ClassDef(self, node): + # ClassDef(name=Name, + # bases=[Expr_1, Expr_2, ..., Expr_n], + # keywords=[], + # body=[], + # decorator_list=[Expr_1, Expr_2, ..., Expr_m]) + return self.generic_visit(node) + + def visit_Return(self, node): + # Return(value=Expr) + return self.generic_visit(node) + + def visit_Delete(self, node): + # Delete(targets=[Expr_1, Expr_2, ..., Expr_n]) + return self.generic_visit(node) + + def visit_Assign(self, node): + # Assign(targets=[Expr_1, Expr_2, ..., Expr_n], + # value=Expr) + return self.generic_visit(node) + + def visit_AugAssign(self, node): + # AugAssign(target=Name, + # op=Add(), + # value=Expr) + return self.generic_visit(node) + + def visit_AnnAssign(self, node): + # AnnAssign(target=Name, + # annotation=TypeName, + # value=Expr, + # simple=number) + return self.generic_visit(node) + + def visit_For(self, node): + # For(target=Expr, + # iter=Expr, + # body=[...], + # orelse=[...]) + return self.generic_visit(node) + + def visit_AsyncFor(self, node): + # AsyncFor(target=Expr, + # iter=Expr, + # body=[...], + # orelse=[...]) + return self.generic_visit(node) + + def visit_While(self, node): + # While(test=Expr, + # body=[...], + # orelse=[...]) + return self.generic_visit(node) + + def visit_If(self, node): + # If(test=Expr, + # body=[...], + # orelse=[...]) + return self.generic_visit(node) + + def visit_With(self, node): + # With(items=[withitem_1, withitem_2, ..., withitem_n], + # body=[...]) + return self.generic_visit(node) + + def visit_AsyncWith(self, node): + # AsyncWith(items=[withitem_1, withitem_2, ..., withitem_n], + # body=[...]) + return self.generic_visit(node) + + def visit_Match(self, node): + # Match(subject=Expr, + # cases=[ + # match_case( + # pattern=pattern, + # guard=Expr, + # body=[...]), + # ... + # ]) + return self.generic_visit(node) + + def visit_Raise(self, node): + # Raise(exc=Expr) + return self.generic_visit(node) + + def visit_Try(self, node): + # Try(body=[...], + # handlers=[ExceptHandler_1, ExceptHandler_2, ..., ExceptHandler_b], + # orelse=[...], + # finalbody=[...]) + return self.generic_visit(node) + + def visit_Assert(self, node): + # Assert(test=Expr) + return self.generic_visit(node) + + def visit_Import(self, node): + # Import(names=[ + # alias( + # name=Identifier, + # asname=Identifier), + # ... + # ]) + return self.generic_visit(node) + + def visit_ImportFrom(self, node): + # ImportFrom(module=Identifier, + # names=[ + # alias( + # name=Identifier, + # asname=Identifier), + # ... + # ], + # level=num + return self.generic_visit(node) + + def visit_Global(self, node): + # Global(names=[Identifier_1, Identifier_2, ..., Identifier_n]) + return self.generic_visit(node) + + def visit_Nonlocal(self, node): + # Nonlocal(names=[Identifier_1, Identifier_2, ..., Identifier_n]) + return self.generic_visit(node) + + def visit_Expr(self, node): + # Expr(value=Expr) + return self.generic_visit(node) + + def visit_Pass(self, node): + # Pass() + return self.generic_visit(node) + + def visit_Break(self, node): + # Break() + return self.generic_visit(node) + + def visit_Continue(self, node): + # Continue() + return self.generic_visit(node) + + ############################################################################ + # Expressions # + ############################################################################ + + def visit_BoolOp(self, node): + # BoolOp(op=And | Or, + # values=[Expr_1, Expr_2, ..., Expr_n]) + return self.generic_visit(node) + + def visit_NamedExpr(self, node): + # NamedExpr(target=Name, + # value=Expr) + return self.generic_visit(node) + + def visit_BinOp(self, node): + # BinOp(left=LExpr + # op=Add | Sub | Mult | MatMult | Div | Mod | Pow | LShift | + # RShift | BitOr | BitXor | BitAnd | FloorDiv + # right=RExpr) + return self.generic_visit(node) + + def visit_UnaryOp(self, node): + # UnaryOp(op=Not | USub | UAdd | Invert, + # operand=Expr) + return self.generic_visit(node) + + def visit_Lambda(self, node): + # Lambda(args=arguments( + # posonlyargs=[], + # args=[ + # arg(arg='a'), + # arg(arg='b')], + # kwonlyargs=[], + # kw_defaults=[], + # defaults=[]), + # body=Expr) + return self.generic_visit(node) + + def visit_IfExp(self, node): + # IfExp(test=TestExpr, + # body=BodyExpr, + # orelse=OrElseExpr) + return self.generic_visit(node) + + def visit_Dict(self, node): + # Dict(keys=[Expr_1, Expr_2, ..., Expr_n], + # values=[Expr_1, Expr_2, ..., Expr_n]) + return self.generic_visit(node) + + def visit_Set(self, node): + # Set(elts=[Expr_1, Expr_2, ..., Expr_n]) + return self.generic_visit(node) + + def visit_ListComp(self, node): + # ListComp(elt=Expr, + # generators=[ + # comprehension( + # target=Expr, + # iter=Expr, + # ifs=[Expr_1, Expr_2, ..., Expr_n], + # is_async=0), + # ... + # ]) + return self.generic_visit(node) + + def visit_SetComp(self, node): + # SetComp(elt=Expr, + # generators=[ + # comprehension( + # target=Expr, + # iter=Expr, + # ifs=[Expr_1, Expr_2, ..., Expr_n], + # is_async=0), + # ... + # ]) + return self.generic_visit(node) + + def visit_DictComp(self, node): + # DictComp(key=KeyExpr, + # value=ValExpr, + # generators=[ + # comprehension( + # target=TargetExpr + # iter=IterExpr, + # ifs=[Expr_1, Expr_2, ..., Expr_n]), + # is_async=0)], + # ... + # ]) + return self.generic_visit(node) + + def visit_GeneratorExp(self, node): + # GeneratorExp(elt=Expr, + # generators=[ + # comprehension( + # target=Expr, + # iter=Expr, + # ifs=[Expr_1, Expr_2, ..., Expr_n], + # is_async=0), + # ... + # ]) + return self.generic_visit(node) + + def visit_Await(self, node): + # Await(value=Expr) + return self.generic_visit(node) + + def visit_Yield(self, node): + # Yield(value=Expr) + return self.generic_visit(node) + + def visit_YieldFrom(self, node): + # YieldFrom(value=Expr) + return self.generic_visit(node) + + def visit_Compare(self, node): + # Compare(left=LExpr, + # ops=[Op_1, Op_2, ..., Op_n], + # comparators=[Expr_1, Expr_2, ..., Expr_n]) + return self.generic_visit(node) + + def visit_Call(self, node): + # Call(func=Expr, + # args=[Expr_1, Expr_2, ..., Expr_n], + # keywords=[ + # keyword( + # arg='d', + # value=Expr), + # ... + # ]) + return self.generic_visit(node) + + def visit_FormattedValue(self, node): + # FormattedValue(value=Expr, + # conversion=-1, + # format_spec=FSExpr) + return self.generic_visit(node) + + def visit_JoinedStr(self, node): + # JoinedStr(values=[Expr_1, Expr_2, ..., Expr_n]) + return self.generic_visit(node) + + def visit_Constant(self, node): + # Constant(value=Expr) + return self.generic_visit(node) + + def visit_Attribute(self, node): + # Attribute(value=Expr, + # attr=Identifier) + return self.generic_visit(node) + + def visit_Subscript(self, node): + # Subscript(value=VExpr, + # slice=SExpr) + return self.generic_visit(node) + + def visit_Starred(self, node): + # Starred(value=Expr) + return self.generic_visit(node) + + def visit_Name(self, node): + # Name(id=Identifier) + return self.generic_visit(node) + + def visit_List(self, node): + # List(elts=[Expr_1, Expr_2, ..., Expr_n]) + return self.generic_visit(node) + + def visit_Tuple(self, node): + # Tuple(elts=[Expr_1, Expr_2, ..., Expr_n]) + return self.generic_visit(node) + + def visit_Slice(self, node): + # Slice(lower=Expr, + # upper=Expr, + # step=Expr) + return self.generic_visit(node) + + ############################################################################ + # Expression Context # + ############################################################################ + + def visit_Load(self, node): + # Load() + return self.generic_visit(node) + + def visit_Store(self, node): + # Store() + return self.generic_visit(node) + + def visit_Del(self, node): + # Del() + return self.generic_visit(node) + + ############################################################################ + # Boolean Operators # + ############################################################################ + + def visit_And(self, node): + # And() + return self.generic_visit(node) + + def visit_Or(self, node): + # Or() + return self.generic_visit(node) + + ############################################################################ + # Binary Operators # + ############################################################################ + + def visit_Add(self, node): + # Add() + return self.generic_visit(node) + + def visit_Sub(self, node): + # Sub() + return self.generic_visit(node) + + def visit_Mult(self, node): + # Mult() + return self.generic_visit(node) + + def visit_MatMult(self, node): + # MatMult() + return self.generic_visit(node) + + def visit_Div(self, node): + # Div() + return self.generic_visit(node) + + def visit_Mod(self, node): + # Mod() + return self.generic_visit(node) + + def visit_Pow(self, node): + # Pow() + return self.generic_visit(node) + + def visit_LShift(self, node): + # LShift() + return self.generic_visit(node) + + def visit_RShift(self, node): + # RShift() + return self.generic_visit(node) + + def visit_BitOr(self, node): + # BitOr() + return self.generic_visit(node) + + def visit_BitXor(self, node): + # BitXor() + return self.generic_visit(node) + + def visit_BitAnd(self, node): + # BitAnd() + return self.generic_visit(node) + + def visit_FloorDiv(self, node): + # FloorDiv() + return self.generic_visit(node) + + ############################################################################ + # Unary Operators # + ############################################################################ + + def visit_Invert(self, node): + # Invert() + return self.generic_visit(node) + + def visit_Not(self, node): + # Not() + return self.generic_visit(node) + + def visit_UAdd(self, node): + # UAdd() + return self.generic_visit(node) + + def visit_USub(self, node): + # USub() + return self.generic_visit(node) + + ############################################################################ + # Comparison Operators # + ############################################################################ + + def visit_Eq(self, node): + # Eq() + return self.generic_visit(node) + + def visit_NotEq(self, node): + # NotEq() + return self.generic_visit(node) + + def visit_Lt(self, node): + # Lt() + return self.generic_visit(node) + + def visit_LtE(self, node): + # LtE() + return self.generic_visit(node) + + def visit_Gt(self, node): + # Gt() + return self.generic_visit(node) + + def visit_GtE(self, node): + # GtE() + return self.generic_visit(node) + + def visit_Is(self, node): + # Is() + return self.generic_visit(node) + + def visit_IsNot(self, node): + # IsNot() + return self.generic_visit(node) + + def visit_In(self, node): + # In() + return self.generic_visit(node) + + def visit_NotIn(self, node): + # NotIn() + return self.generic_visit(node) + + ############################################################################ + # Exception Handler # + ############################################################################ + + def visit_ExceptionHandler(self, node): + # ExceptHandler(type=Expr, + # name=Identifier, + # body=[...]) + return self.generic_visit(node) + + ############################################################################ + # Matching Patterns # + ############################################################################ + + def visit_MatchValue(self, node): + # MatchValue(value=Expr) + return self.generic_visit(node) + + def visit_MatchSingleton(self, node): + # MatchSingleton(value=Constant) + return self.generic_visit(node) + + def visit_MatchSequence(self, node): + # MatchSequence(patterns=[pattern_1, pattern_2, ..., pattern_n]) + return self.generic_visit(node) + + def visit_MatchMapping(self, node): + # MatchMapping(keys=[Expr_1, Expr_2, ..., Expr_n], + # patterns=[pattern_1, pattern_2, ..., pattern_m], + # rest=Identifier) + return self.generic_visit(node) + + def visit_MatchClass(self, node): + # MatchClass(cls=Expr, + # patterns=[pattern_1, pattern_2, ...], + # kwd_attrs=[Identifier_1, Identifier_2, ...], + # kwd_patterns=[pattern_1, pattern_2, ...]) + return self.generic_visit(node) + + def visit_MatchStar(self, node): + # MatchStar(name=Identifier) + return self.generic_visit(node) + + def visit_MatchAs(self, node): + # MatchAs(pattern=pattern, + # name=Identifier) + return self.generic_visit(node) + + def visit_MatchOr(self, node): + # MatchOr(patterns=[pattern_1, pattern_2, ...]) + return self.generic_visit(node) + + ############################################################################ + # Type Ignore # + ############################################################################ + + def visit_TypeIgnore(self, node): + # TypeIgnore(tag=string) + return self.generic_visit(node) + + ############################################################################ + # Miscellaneous # + ############################################################################ + + def visit_comprehension(self, node): + # comprehension(target=Expr, + # iter=Expr, + # ifs=[Expr_1, Expr_2, ..., Expr_n], + # is_async=0) + return self.generic_visit(node) + + def visit_arguments(self, node): + # arguments(posonlyargs=[], + # args=[], + # vararg=arg, + # kwonlyargs=[], + # kw_defaults=[], + # kwarg=arg, + # defaults=[]), + return self.generic_visit(node) + + def visit_arg(self, node): + # arg(arg=Identifier, + # annotation=Expr, + # type_comment='') + return self.generic_visit(node) + + def visit_keyword(self, node): + # keyword(arg=Identifier, + # value=Expr) + return self.generic_visit(node) + + def visit_alias(self, node): + # alias(name=Identifier, + # asname=Identifier) + return self.generic_visit(node) + + def visit_withitem(self, node): + # withitem(context_expr=Expr, + # optional_vars=Expr) + return self.generic_visit(node) + + def visit_match_case(self, node): + # match_case(pattern=pattern, + # guard=Expr, + # body=[...]) + return self.generic_visit(node) diff --git a/yapf/pyparser/split_penalty_visitor.py b/yapf/pyparser/split_penalty_visitor.py new file mode 100644 index 000000000..8cd25c959 --- /dev/null +++ b/yapf/pyparser/split_penalty_visitor.py @@ -0,0 +1,913 @@ +# Copyright 2022 Bill Wendling, All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import ast + +from yapf.pyparser import pyparser_utils as pyutils +from yapf.yapflib import split_penalty +from yapf.yapflib import style +from yapf.yapflib import subtypes + + +class SplitPenalty(ast.NodeVisitor): + """Compute split penalties between tokens.""" + + def __init__(self, logical_lines): + super(SplitPenalty, self).__init__() + self.logical_lines = logical_lines + + # We never want to split before a colon or comma. + for logical_line in logical_lines: + for token in logical_line.tokens: + if token.value in frozenset({',', ':'}): + token.split_penalty = split_penalty.UNBREAKABLE + + def _GetTokens(self, node): + return pyutils.GetLogicalLine(self.logical_lines, node) + + ############################################################################ + # Statements # + ############################################################################ + + def visit_FunctionDef(self, node): + # FunctionDef(name=Name, + # args=arguments( + # posonlyargs=[], + # args=[], + # vararg=[], + # kwonlyargs=[], + # kw_defaults=[], + # defaults=[]), + # body=[...], + # decorator_list=[Call_1, Call_2, ..., Call_n], + # keywords=[]) + tokens = self._GetTokens(node) + + for decorator in node.decorator_list: + # The decorator token list begins after the '@'. The body of the decorator + # is formatted like a normal "call." + decorator_range = self._GetTokens(decorator) + # Don't split after the '@'. + decorator_range[0].split_penalty = split_penalty.UNBREAKABLE + + for token in tokens[1:]: + if token.value == '(': + break + _SetPenalty(token, split_penalty.UNBREAKABLE) + + if node.returns: + start_index = pyutils.GetTokenIndex(tokens, + pyutils.TokenStart(node.returns)) + _IncreasePenalty(tokens[start_index - 1:start_index + 1], + split_penalty.VERY_STRONGLY_CONNECTED) + end_index = pyutils.GetTokenIndex(tokens, pyutils.TokenEnd(node.returns)) + _IncreasePenalty(tokens[start_index + 1:end_index], + split_penalty.STRONGLY_CONNECTED) + + return self.generic_visit(node) + + def visit_AsyncFunctionDef(self, node): + # AsyncFunctionDef(name=Name, + # args=arguments( + # posonlyargs=[], + # args=[], + # vararg=[], + # kwonlyargs=[], + # kw_defaults=[], + # defaults=[]), + # body=[...], + # decorator_list=[Expr_1, Expr_2, ..., Expr_n], + # keywords=[]) + return self.visit_FunctionDef(node) + + def visit_ClassDef(self, node): + # ClassDef(name=Name, + # bases=[Expr_1, Expr_2, ..., Expr_n], + # keywords=[], + # body=[], + # decorator_list=[Expr_1, Expr_2, ..., Expr_m]) + for base in node.bases: + tokens = self._GetTokens(base) + _IncreasePenalty(tokens[1:], split_penalty.EXPR) + + for decorator in node.decorator_list: + # Don't split after the '@'. + tokens = self._GetTokens(decorator) + tokens[0].split_penalty = split_penalty.UNBREAKABLE + + return self.generic_visit(node) + + def visit_Return(self, node): + # Return(value=Expr) + tokens = self._GetTokens(node) + _IncreasePenalty(tokens[1:], split_penalty.EXPR) + + return self.generic_visit(node) + + def visit_Delete(self, node): + # Delete(targets=[Expr_1, Expr_2, ..., Expr_n]) + for target in node.targets: + tokens = self._GetTokens(target) + _IncreasePenalty(tokens[1:], split_penalty.EXPR) + + return self.generic_visit(node) + + def visit_Assign(self, node): + # Assign(targets=[Expr_1, Expr_2, ..., Expr_n], + # value=Expr) + tokens = self._GetTokens(node) + _IncreasePenalty(tokens[1:], split_penalty.EXPR) + + return self.generic_visit(node) + + def visit_AugAssign(self, node): + # AugAssign(target=Name, + # op=Add(), + # value=Expr) + return self.generic_visit(node) + + def visit_AnnAssign(self, node): + # AnnAssign(target=Expr, + # annotation=TypeName, + # value=Expr, + # simple=number) + return self.generic_visit(node) + + def visit_For(self, node): + # For(target=Expr, + # iter=Expr, + # body=[...], + # orelse=[...]) + return self.generic_visit(node) + + def visit_AsyncFor(self, node): + # AsyncFor(target=Expr, + # iter=Expr, + # body=[...], + # orelse=[...]) + return self.generic_visit(node) + + def visit_While(self, node): + # While(test=Expr, + # body=[...], + # orelse=[...]) + return self.generic_visit(node) + + def visit_If(self, node): + # If(test=Expr, + # body=[...], + # orelse=[...]) + return self.generic_visit(node) + + def visit_With(self, node): + # With(items=[withitem_1, withitem_2, ..., withitem_n], + # body=[...]) + return self.generic_visit(node) + + def visit_AsyncWith(self, node): + # AsyncWith(items=[withitem_1, withitem_2, ..., withitem_n], + # body=[...]) + return self.generic_visit(node) + + def visit_Match(self, node): + # Match(subject=Expr, + # cases=[ + # match_case( + # pattern=pattern, + # guard=Expr, + # body=[...]), + # ... + # ]) + return self.generic_visit(node) + + def visit_Raise(self, node): + # Raise(exc=Expr) + return self.generic_visit(node) + + def visit_Try(self, node): + # Try(body=[...], + # handlers=[ExceptHandler_1, ExceptHandler_2, ..., ExceptHandler_b], + # orelse=[...], + # finalbody=[...]) + return self.generic_visit(node) + + def visit_Assert(self, node): + # Assert(test=Expr) + return self.generic_visit(node) + + def visit_Import(self, node): + # Import(names=[ + # alias( + # name=Identifier, + # asname=Identifier), + # ... + # ]) + return self.generic_visit(node) + + def visit_ImportFrom(self, node): + # ImportFrom(module=Identifier, + # names=[ + # alias( + # name=Identifier, + # asname=Identifier), + # ... + # ], + # level=num + return self.generic_visit(node) + + def visit_Global(self, node): + # Global(names=[Identifier_1, Identifier_2, ..., Identifier_n]) + return self.generic_visit(node) + + def visit_Nonlocal(self, node): + # Nonlocal(names=[Identifier_1, Identifier_2, ..., Identifier_n]) + return self.generic_visit(node) + + def visit_Expr(self, node): + # Expr(value=Expr) + return self.generic_visit(node) + + def visit_Pass(self, node): + # Pass() + return self.generic_visit(node) + + def visit_Break(self, node): + # Break() + return self.generic_visit(node) + + def visit_Continue(self, node): + # Continue() + return self.generic_visit(node) + + ############################################################################ + # Expressions # + ############################################################################ + + def visit_BoolOp(self, node): + # BoolOp(op=And | Or, + # values=[Expr_1, Expr_2, ..., Expr_n]) + tokens = self._GetTokens(node) + _IncreasePenalty(tokens[1:], split_penalty.EXPR) + + # Lower the split penalty to allow splitting before or after the logical + # operator. + split_before_operator = style.Get('SPLIT_BEFORE_LOGICAL_OPERATOR') + operator_indices = [ + pyutils.GetNextTokenIndex(tokens, pyutils.TokenEnd(value)) + for value in node.values[:-1] + ] + for operator_index in operator_indices: + if not split_before_operator: + operator_index += 1 + _DecreasePenalty(tokens[operator_index], split_penalty.EXPR * 2) + + return self.generic_visit(node) + + def visit_NamedExpr(self, node): + # NamedExpr(target=Name, + # value=Expr) + tokens = self._GetTokens(node) + _IncreasePenalty(tokens[1:], split_penalty.EXPR) + + return self.generic_visit(node) + + def visit_BinOp(self, node): + # BinOp(left=LExpr + # op=Add | Sub | Mult | MatMult | Div | Mod | Pow | LShift | + # RShift | BitOr | BitXor | BitAnd | FloorDiv + # right=RExpr) + tokens = self._GetTokens(node) + + _IncreasePenalty(tokens[1:], split_penalty.EXPR) + + # Lower the split penalty to allow splitting before or after the arithmetic + # operator. + operator_index = pyutils.GetNextTokenIndex(tokens, + pyutils.TokenEnd(node.left)) + if not style.Get('SPLIT_BEFORE_ARITHMETIC_OPERATOR'): + operator_index += 1 + + _DecreasePenalty(tokens[operator_index], split_penalty.EXPR * 2) + + return self.generic_visit(node) + + def visit_UnaryOp(self, node): + # UnaryOp(op=Not | USub | UAdd | Invert, + # operand=Expr) + tokens = self._GetTokens(node) + _IncreasePenalty(tokens[1:], split_penalty.EXPR) + _IncreasePenalty(tokens[1], style.Get('SPLIT_PENALTY_AFTER_UNARY_OPERATOR')) + + return self.generic_visit(node) + + def visit_Lambda(self, node): + # Lambda(args=arguments( + # posonlyargs=[arg(...), arg(...), ..., arg(...)], + # args=[arg(...), arg(...), ..., arg(...)], + # kwonlyargs=[arg(...), arg(...), ..., arg(...)], + # kw_defaults=[arg(...), arg(...), ..., arg(...)], + # defaults=[arg(...), arg(...), ..., arg(...)]), + # body=Expr) + tokens = self._GetTokens(node) + _IncreasePenalty(tokens[1:], split_penalty.LAMBDA) + + if style.Get('ALLOW_MULTILINE_LAMBDAS'): + _SetPenalty(self._GetTokens(node.body), split_penalty.MULTIPLINE_LAMBDA) + + return self.generic_visit(node) + + def visit_IfExp(self, node): + # IfExp(test=TestExpr, + # body=BodyExpr, + # orelse=OrElseExpr) + tokens = self._GetTokens(node) + _IncreasePenalty(tokens[1:], split_penalty.EXPR) + + return self.generic_visit(node) + + def visit_Dict(self, node): + # Dict(keys=[Expr_1, Expr_2, ..., Expr_n], + # values=[Expr_1, Expr_2, ..., Expr_n]) + tokens = self._GetTokens(node) + + # The keys should be on a single line if at all possible. + for key in node.keys: + subrange = pyutils.GetTokensInSubRange(tokens, key) + _IncreasePenalty(subrange[1:], split_penalty.DICT_KEY_EXPR) + + for value in node.values: + subrange = pyutils.GetTokensInSubRange(tokens, value) + _IncreasePenalty(subrange[1:], split_penalty.DICT_VALUE_EXPR) + + return self.generic_visit(node) + + def visit_Set(self, node): + # Set(elts=[Expr_1, Expr_2, ..., Expr_n]) + tokens = self._GetTokens(node) + for element in node.elts: + subrange = pyutils.GetTokensInSubRange(tokens, element) + _IncreasePenalty(subrange[1:], split_penalty.EXPR) + + return self.generic_visit(node) + + def visit_ListComp(self, node): + # ListComp(elt=Expr, + # generators=[ + # comprehension( + # target=Expr, + # iter=Expr, + # ifs=[Expr_1, Expr_2, ..., Expr_n], + # is_async=0), + # ... + # ]) + tokens = self._GetTokens(node) + element = pyutils.GetTokensInSubRange(tokens, node.elt) + _IncreasePenalty(element[1:], split_penalty.EXPR) + + for comp in node.generators: + subrange = pyutils.GetTokensInSubRange(tokens, comp.iter) + _IncreasePenalty(subrange[1:], split_penalty.EXPR) + + for if_expr in comp.ifs: + subrange = pyutils.GetTokensInSubRange(tokens, if_expr) + _IncreasePenalty(subrange[1:], split_penalty.EXPR) + + return self.generic_visit(node) + + def visit_SetComp(self, node): + # SetComp(elt=Expr, + # generators=[ + # comprehension( + # target=Expr, + # iter=Expr, + # ifs=[Expr_1, Expr_2, ..., Expr_n], + # is_async=0), + # ... + # ]) + tokens = self._GetTokens(node) + element = pyutils.GetTokensInSubRange(tokens, node.elt) + _IncreasePenalty(element[1:], split_penalty.EXPR) + + for comp in node.generators: + subrange = pyutils.GetTokensInSubRange(tokens, comp.iter) + _IncreasePenalty(subrange[1:], split_penalty.EXPR) + + for if_expr in comp.ifs: + subrange = pyutils.GetTokensInSubRange(tokens, if_expr) + _IncreasePenalty(subrange[1:], split_penalty.EXPR) + + return self.generic_visit(node) + + def visit_DictComp(self, node): + # DictComp(key=KeyExpr, + # value=ValExpr, + # generators=[ + # comprehension( + # target=TargetExpr + # iter=IterExpr, + # ifs=[Expr_1, Expr_2, ..., Expr_n]), + # is_async=0)], + # ... + # ]) + tokens = self._GetTokens(node) + key = pyutils.GetTokensInSubRange(tokens, node.key) + _IncreasePenalty(key[1:], split_penalty.EXPR) + + value = pyutils.GetTokensInSubRange(tokens, node.value) + _IncreasePenalty(value[1:], split_penalty.EXPR) + + for comp in node.generators: + subrange = pyutils.GetTokensInSubRange(tokens, comp.iter) + _IncreasePenalty(subrange[1:], split_penalty.EXPR) + + for if_expr in comp.ifs: + subrange = pyutils.GetTokensInSubRange(tokens, if_expr) + _IncreasePenalty(subrange[1:], split_penalty.EXPR) + + return self.generic_visit(node) + + def visit_GeneratorExp(self, node): + # GeneratorExp(elt=Expr, + # generators=[ + # comprehension( + # target=Expr, + # iter=Expr, + # ifs=[Expr_1, Expr_2, ..., Expr_n], + # is_async=0), + # ... + # ]) + tokens = self._GetTokens(node) + element = pyutils.GetTokensInSubRange(tokens, node.elt) + _IncreasePenalty(element[1:], split_penalty.EXPR) + + for comp in node.generators: + subrange = pyutils.GetTokensInSubRange(tokens, comp.iter) + _IncreasePenalty(subrange[1:], split_penalty.EXPR) + + for if_expr in comp.ifs: + subrange = pyutils.GetTokensInSubRange(tokens, if_expr) + _IncreasePenalty(subrange[1:], split_penalty.EXPR) + + return self.generic_visit(node) + + def visit_Await(self, node): + # Await(value=Expr) + tokens = self._GetTokens(node) + _IncreasePenalty(tokens[1:], split_penalty.EXPR) + + return self.generic_visit(node) + + def visit_Yield(self, node): + # Yield(value=Expr) + tokens = self._GetTokens(node) + _IncreasePenalty(tokens[1:], split_penalty.EXPR) + + return self.generic_visit(node) + + def visit_YieldFrom(self, node): + # YieldFrom(value=Expr) + tokens = self._GetTokens(node) + _IncreasePenalty(tokens[1:], split_penalty.EXPR) + tokens[2].split_penalty = split_penalty.UNBREAKABLE + + return self.generic_visit(node) + + def visit_Compare(self, node): + # Compare(left=LExpr, + # ops=[Op_1, Op_2, ..., Op_n], + # comparators=[Expr_1, Expr_2, ..., Expr_n]) + tokens = self._GetTokens(node) + _IncreasePenalty(tokens[1:], split_penalty.EXPR) + + operator_indices = [ + pyutils.GetNextTokenIndex(tokens, pyutils.TokenEnd(node.left)) + ] + [ + pyutils.GetNextTokenIndex(tokens, pyutils.TokenEnd(comparator)) + for comparator in node.comparators[:-1] + ] + split_before = style.Get('SPLIT_BEFORE_ARITHMETIC_OPERATOR') + + for operator_index in operator_indices: + if not split_before: + operator_index += 1 + _DecreasePenalty(tokens[operator_index], split_penalty.EXPR * 2) + + return self.generic_visit(node) + + def visit_Call(self, node): + # Call(func=Expr, + # args=[Expr_1, Expr_2, ..., Expr_n], + # keywords=[ + # keyword( + # arg='d', + # value=Expr), + # ... + # ]) + tokens = self._GetTokens(node) + + # Don't never split before the opening parenthesis. + paren_index = pyutils.GetNextTokenIndex(tokens, pyutils.TokenEnd(node.func)) + _IncreasePenalty(tokens[paren_index], split_penalty.UNBREAKABLE) + + for arg in node.args: + subrange = pyutils.GetTokensInSubRange(tokens, arg) + _IncreasePenalty(subrange[1:], split_penalty.EXPR) + + return self.generic_visit(node) + + def visit_FormattedValue(self, node): + # FormattedValue(value=Expr, + # conversion=-1) + return node # Ignore formatted values. + + def visit_JoinedStr(self, node): + # JoinedStr(values=[Expr_1, Expr_2, ..., Expr_n]) + return self.generic_visit(node) + + def visit_Constant(self, node): + # Constant(value=Expr) + return self.generic_visit(node) + + def visit_Attribute(self, node): + # Attribute(value=Expr, + # attr=Identifier) + tokens = self._GetTokens(node) + split_before = style.Get('SPLIT_BEFORE_DOT') + dot_indices = pyutils.GetNextTokenIndex(tokens, + pyutils.TokenEnd(node.value)) + + if not split_before: + dot_indices += 1 + _IncreasePenalty(tokens[dot_indices], split_penalty.VERY_STRONGLY_CONNECTED) + + return self.generic_visit(node) + + def visit_Subscript(self, node): + # Subscript(value=ValueExpr, + # slice=SliceExpr) + tokens = self._GetTokens(node) + + # Don't split before the opening bracket of a subscript. + bracket_index = pyutils.GetNextTokenIndex(tokens, + pyutils.TokenEnd(node.value)) + _IncreasePenalty(tokens[bracket_index], split_penalty.UNBREAKABLE) + + return self.generic_visit(node) + + def visit_Starred(self, node): + # Starred(value=Expr) + return self.generic_visit(node) + + def visit_Name(self, node): + # Name(id=Identifier) + tokens = self._GetTokens(node) + _IncreasePenalty(tokens[1:], split_penalty.UNBREAKABLE) + + return self.generic_visit(node) + + def visit_List(self, node): + # List(elts=[Expr_1, Expr_2, ..., Expr_n]) + tokens = self._GetTokens(node) + + for element in node.elts: + subrange = pyutils.GetTokensInSubRange(tokens, element) + _IncreasePenalty(subrange[1:], split_penalty.EXPR) + _DecreasePenalty(subrange[0], split_penalty.EXPR // 2) + + return self.generic_visit(node) + + def visit_Tuple(self, node): + # Tuple(elts=[Expr_1, Expr_2, ..., Expr_n]) + tokens = self._GetTokens(node) + + for element in node.elts: + subrange = pyutils.GetTokensInSubRange(tokens, element) + _IncreasePenalty(subrange[1:], split_penalty.EXPR) + _DecreasePenalty(subrange[0], split_penalty.EXPR // 2) + + return self.generic_visit(node) + + def visit_Slice(self, node): + # Slice(lower=Expr, + # upper=Expr, + # step=Expr) + tokens = self._GetTokens(node) + + if hasattr(node, 'lower') and node.lower: + subrange = pyutils.GetTokensInSubRange(tokens, node.lower) + _IncreasePenalty(subrange, split_penalty.EXPR) + _DecreasePenalty(subrange[0], split_penalty.EXPR // 2) + + if hasattr(node, 'upper') and node.upper: + colon_index = pyutils.GetPrevTokenIndex(tokens, + pyutils.TokenStart(node.upper)) + _IncreasePenalty(tokens[colon_index], split_penalty.UNBREAKABLE) + subrange = pyutils.GetTokensInSubRange(tokens, node.upper) + _IncreasePenalty(subrange, split_penalty.EXPR) + _DecreasePenalty(subrange[0], split_penalty.EXPR // 2) + + if hasattr(node, 'step') and node.step: + colon_index = pyutils.GetPrevTokenIndex(tokens, + pyutils.TokenStart(node.step)) + _IncreasePenalty(tokens[colon_index], split_penalty.UNBREAKABLE) + subrange = pyutils.GetTokensInSubRange(tokens, node.step) + _IncreasePenalty(subrange, split_penalty.EXPR) + _DecreasePenalty(subrange[0], split_penalty.EXPR // 2) + + return self.generic_visit(node) + + ############################################################################ + # Expression Context # + ############################################################################ + + def visit_Load(self, node): + # Load() + return self.generic_visit(node) + + def visit_Store(self, node): + # Store() + return self.generic_visit(node) + + def visit_Del(self, node): + # Del() + return self.generic_visit(node) + + ############################################################################ + # Boolean Operators # + ############################################################################ + + def visit_And(self, node): + # And() + return self.generic_visit(node) + + def visit_Or(self, node): + # Or() + return self.generic_visit(node) + + ############################################################################ + # Binary Operators # + ############################################################################ + + def visit_Add(self, node): + # Add() + return self.generic_visit(node) + + def visit_Sub(self, node): + # Sub() + return self.generic_visit(node) + + def visit_Mult(self, node): + # Mult() + return self.generic_visit(node) + + def visit_MatMult(self, node): + # MatMult() + return self.generic_visit(node) + + def visit_Div(self, node): + # Div() + return self.generic_visit(node) + + def visit_Mod(self, node): + # Mod() + return self.generic_visit(node) + + def visit_Pow(self, node): + # Pow() + return self.generic_visit(node) + + def visit_LShift(self, node): + # LShift() + return self.generic_visit(node) + + def visit_RShift(self, node): + # RShift() + return self.generic_visit(node) + + def visit_BitOr(self, node): + # BitOr() + return self.generic_visit(node) + + def visit_BitXor(self, node): + # BitXor() + return self.generic_visit(node) + + def visit_BitAnd(self, node): + # BitAnd() + return self.generic_visit(node) + + def visit_FloorDiv(self, node): + # FloorDiv() + return self.generic_visit(node) + + ############################################################################ + # Unary Operators # + ############################################################################ + + def visit_Invert(self, node): + # Invert() + return self.generic_visit(node) + + def visit_Not(self, node): + # Not() + return self.generic_visit(node) + + def visit_UAdd(self, node): + # UAdd() + return self.generic_visit(node) + + def visit_USub(self, node): + # USub() + return self.generic_visit(node) + + ############################################################################ + # Comparison Operators # + ############################################################################ + + def visit_Eq(self, node): + # Eq() + return self.generic_visit(node) + + def visit_NotEq(self, node): + # NotEq() + return self.generic_visit(node) + + def visit_Lt(self, node): + # Lt() + return self.generic_visit(node) + + def visit_LtE(self, node): + # LtE() + return self.generic_visit(node) + + def visit_Gt(self, node): + # Gt() + return self.generic_visit(node) + + def visit_GtE(self, node): + # GtE() + return self.generic_visit(node) + + def visit_Is(self, node): + # Is() + return self.generic_visit(node) + + def visit_IsNot(self, node): + # IsNot() + return self.generic_visit(node) + + def visit_In(self, node): + # In() + return self.generic_visit(node) + + def visit_NotIn(self, node): + # NotIn() + return self.generic_visit(node) + + ############################################################################ + # Exception Handler # + ############################################################################ + + def visit_ExceptionHandler(self, node): + # ExceptHandler(type=Expr, + # name=Identifier, + # body=[...]) + return self.generic_visit(node) + + ############################################################################ + # Matching Patterns # + ############################################################################ + + def visit_MatchValue(self, node): + # MatchValue(value=Expr) + return self.generic_visit(node) + + def visit_MatchSingleton(self, node): + # MatchSingleton(value=Constant) + return self.generic_visit(node) + + def visit_MatchSequence(self, node): + # MatchSequence(patterns=[pattern_1, pattern_2, ..., pattern_n]) + return self.generic_visit(node) + + def visit_MatchMapping(self, node): + # MatchMapping(keys=[Expr_1, Expr_2, ..., Expr_n], + # patterns=[pattern_1, pattern_2, ..., pattern_m], + # rest=Identifier) + return self.generic_visit(node) + + def visit_MatchClass(self, node): + # MatchClass(cls=Expr, + # patterns=[pattern_1, pattern_2, ...], + # kwd_attrs=[Identifier_1, Identifier_2, ...], + # kwd_patterns=[pattern_1, pattern_2, ...]) + return self.generic_visit(node) + + def visit_MatchStar(self, node): + # MatchStar(name=Identifier) + return self.generic_visit(node) + + def visit_MatchAs(self, node): + # MatchAs(pattern=pattern, + # name=Identifier) + return self.generic_visit(node) + + def visit_MatchOr(self, node): + # MatchOr(patterns=[pattern_1, pattern_2, ...]) + return self.generic_visit(node) + + ############################################################################ + # Type Ignore # + ############################################################################ + + def visit_TypeIgnore(self, node): + # TypeIgnore(tag=string) + return self.generic_visit(node) + + ############################################################################ + # Miscellaneous # + ############################################################################ + + def visit_comprehension(self, node): + # comprehension(target=Expr, + # iter=Expr, + # ifs=[Expr_1, Expr_2, ..., Expr_n], + # is_async=0) + return self.generic_visit(node) + + def visit_arguments(self, node): + # arguments(posonlyargs=[arg_1, arg_2, ..., arg_a], + # args=[arg_1, arg_2, ..., arg_b], + # vararg=arg, + # kwonlyargs=[arg_1, arg_2, ..., arg_c], + # kw_defaults=[arg_1, arg_2, ..., arg_d], + # kwarg=arg, + # defaults=[Expr_1, Expr_2, ..., Expr_n]) + return self.generic_visit(node) + + def visit_arg(self, node): + # arg(arg=Identifier, + # annotation=Expr, + # type_comment='') + tokens = self._GetTokens(node) + + # Process any annotations. + if hasattr(node, 'annotation') and node.annotation: + annotation = node.annotation + subrange = pyutils.GetTokensInSubRange(tokens, annotation) + _IncreasePenalty(subrange, split_penalty.ANNOTATION) + + return self.generic_visit(node) + + def visit_keyword(self, node): + # keyword(arg=Identifier, + # value=Expr) + return self.generic_visit(node) + + def visit_alias(self, node): + # alias(name=Identifier, + # asname=Identifier) + return self.generic_visit(node) + + def visit_withitem(self, node): + # withitem(context_expr=Expr, + # optional_vars=Expr) + return self.generic_visit(node) + + def visit_match_case(self, node): + # match_case(pattern=pattern, + # guard=Expr, + # body=[...]) + return self.generic_visit(node) + + +def _IncreasePenalty(tokens, amt): + if not isinstance(tokens, list): + tokens = [tokens] + for token in tokens: + token.split_penalty += amt + + +def _DecreasePenalty(tokens, amt): + if not isinstance(tokens, list): + tokens = [tokens] + for token in tokens: + token.split_penalty -= amt + + +def _SetPenalty(tokens, amt): + if not isinstance(tokens, list): + tokens = [tokens] + for token in tokens: + token.split_penalty = amt diff --git a/yapf/pytree/__init__.py b/yapf/pytree/__init__.py new file mode 100644 index 000000000..8aa1c83c4 --- /dev/null +++ b/yapf/pytree/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2021 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/yapf/yapflib/blank_line_calculator.py b/yapf/pytree/blank_line_calculator.py similarity index 74% rename from yapf/yapflib/blank_line_calculator.py rename to yapf/pytree/blank_line_calculator.py index e02fded8f..32faaa2f4 100644 --- a/yapf/yapflib/blank_line_calculator.py +++ b/yapf/pytree/blank_line_calculator.py @@ -1,4 +1,4 @@ -# Copyright 2015-2016 Google Inc. All Rights Reserved. +# Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,11 +22,11 @@ newlines: The number of newlines required before the node. """ -from lib2to3 import pytree +from yapf_third_party._ylib2to3.pgen2 import token as grammar_token -from yapf.yapflib import py3compat -from yapf.yapflib import pytree_utils -from yapf.yapflib import pytree_visitor +from yapf.pytree import pytree_utils +from yapf.pytree import pytree_visitor +from yapf.yapflib import style _NO_BLANK_LINES = 1 _ONE_BLANK_LINE = 2 @@ -65,15 +65,15 @@ def __init__(self): def Visit_simple_stmt(self, node): # pylint: disable=invalid-name self.DefaultNodeVisit(node) - if pytree_utils.NodeName(node.children[0]) == 'COMMENT': + if node.children[0].type == grammar_token.COMMENT: self.last_comment_lineno = node.children[0].lineno def Visit_decorator(self, node): # pylint: disable=invalid-name if (self.last_comment_lineno and self.last_comment_lineno == node.children[0].lineno - 1): - self._SetNumNewlines(node.children[0], _NO_BLANK_LINES) + _SetNumNewlines(node.children[0], _NO_BLANK_LINES) else: - self._SetNumNewlines(node.children[0], self._GetNumNewlines(node)) + _SetNumNewlines(node.children[0], self._GetNumNewlines(node)) for child in node.children: self.Visit(child) self.last_was_decorator = True @@ -92,11 +92,11 @@ def Visit_funcdef(self, node): # pylint: disable=invalid-name self.last_was_class_or_function = False index = self._SetBlankLinesBetweenCommentAndClassFunc(node) if _AsyncFunction(node): - # Move the number of blank lines to the async keyword. - num_newlines = pytree_utils.GetNodeAnnotation( - node.children[0], pytree_utils.Annotation.NEWLINES) - self._SetNumNewlines(node.prev_sibling, num_newlines) - self._SetNumNewlines(node.children[0], None) + index = self._SetBlankLinesBetweenCommentAndClassFunc( + node.prev_sibling.parent) + _SetNumNewlines(node.children[0], None) + else: + index = self._SetBlankLinesBetweenCommentAndClassFunc(node) self.last_was_decorator = False self.function_level += 1 for child in node.children[index:]: @@ -115,9 +115,8 @@ def DefaultNodeVisit(self, node): """ if self.last_was_class_or_function: if pytree_utils.NodeName(node) in _PYTHON_STATEMENTS: - leaf = _GetFirstChildLeaf(node) - if pytree_utils.NodeName(leaf) != 'COMMENT': - self._SetNumNewlines(leaf, self._GetNumNewlines(leaf)) + leaf = pytree_utils.FirstLeafNode(node) + _SetNumNewlines(leaf, self._GetNumNewlines(leaf)) self.last_was_class_or_function = False super(_BlankLineCalculator, self).DefaultNodeVisit(node) @@ -138,46 +137,41 @@ def _SetBlankLinesBetweenCommentAndClassFunc(self, node): # Standalone comments are wrapped in a simple_stmt node with the comment # node as its only child. self.Visit(node.children[index].children[0]) - self._SetNumNewlines(node.children[index].children[0], _ONE_BLANK_LINE) + if not self.last_was_decorator: + _SetNumNewlines(node.children[index].children[0], _ONE_BLANK_LINE) index += 1 - if (index and node.children[index].lineno - 1 == - node.children[index - 1].children[0].lineno): - self._SetNumNewlines(node.children[index], _NO_BLANK_LINES) + if (index and node.children[index].lineno - 1 + == node.children[index - 1].children[0].lineno): + _SetNumNewlines(node.children[index], _NO_BLANK_LINES) else: if self.last_comment_lineno + 1 == node.children[index].lineno: num_newlines = _NO_BLANK_LINES else: num_newlines = self._GetNumNewlines(node) - self._SetNumNewlines(node.children[index], num_newlines) + _SetNumNewlines(node.children[index], num_newlines) return index def _GetNumNewlines(self, node): if self.last_was_decorator: return _NO_BLANK_LINES elif self._IsTopLevel(node): - return _TWO_BLANK_LINES + return 1 + style.Get('BLANK_LINES_AROUND_TOP_LEVEL_DEFINITION') return _ONE_BLANK_LINE - def _SetNumNewlines(self, node, num_newlines): - pytree_utils.SetNodeAnnotation(node, pytree_utils.Annotation.NEWLINES, - num_newlines) - def _IsTopLevel(self, node): return (not (self.class_level or self.function_level) and _StartsInZerothColumn(node)) +def _SetNumNewlines(node, num_newlines): + pytree_utils.SetNodeAnnotation(node, pytree_utils.Annotation.NEWLINES, + num_newlines) + + def _StartsInZerothColumn(node): - return (_GetFirstChildLeaf(node).column == 0 or + return (pytree_utils.FirstLeafNode(node).column == 0 or (_AsyncFunction(node) and node.prev_sibling.column == 0)) def _AsyncFunction(node): - return (py3compat.PY3 and node.prev_sibling and - pytree_utils.NodeName(node.prev_sibling) == 'ASYNC') - - -def _GetFirstChildLeaf(node): - if isinstance(node, pytree.Leaf): - return node - return _GetFirstChildLeaf(node.children[0]) + return (node.prev_sibling and node.prev_sibling.type == grammar_token.ASYNC) diff --git a/yapf/yapflib/comment_splicer.py b/yapf/pytree/comment_splicer.py similarity index 72% rename from yapf/yapflib/comment_splicer.py rename to yapf/pytree/comment_splicer.py index d7c9536c1..a9180f37c 100644 --- a/yapf/yapflib/comment_splicer.py +++ b/yapf/pytree/comment_splicer.py @@ -1,4 +1,4 @@ -# Copyright 2015-2016 Google Inc. All Rights Reserved. +# Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -21,11 +21,11 @@ SpliceComments(): the main function exported by this module. """ -from lib2to3 import pygram -from lib2to3 import pytree -from lib2to3.pgen2 import token +from yapf_third_party._ylib2to3 import pygram +from yapf_third_party._ylib2to3 import pytree +from yapf_third_party._ylib2to3.pgen2 import token -from yapf.yapflib import pytree_utils +from yapf.pytree import pytree_utils def SpliceComments(tree): @@ -44,6 +44,7 @@ def SpliceComments(tree): _AnnotateIndents(tree) def _VisitNodeRec(node): + """Recursively visit each node to splice comments into the AST.""" # This loop may insert into node.children, so we'll iterate over a copy. for child in node.children[:]: if isinstance(child, pytree.Node): @@ -61,6 +62,8 @@ def _VisitNodeRec(node): # child in the next iteration... child_prefix = child.prefix.lstrip('\n') prefix_indent = child_prefix[:child_prefix.find('#')] + if '\n' in prefix_indent: + prefix_indent = prefix_indent[prefix_indent.rfind('\n') + 1:] child.prefix = '' if child.type == token.NEWLINE: @@ -69,79 +72,64 @@ def _VisitNodeRec(node): # We can't just insert it before the NEWLINE node, because as a # result of the way pytrees are organized, this node can be under # an inappropriate parent. - comment_column -= len(comment_prefix) - comment_column += len(comment_prefix) - len(comment_prefix.lstrip()) + comment_column -= len(comment_prefix.lstrip()) pytree_utils.InsertNodesAfter( _CreateCommentsFromPrefix( comment_prefix, comment_lineno, comment_column, - standalone=False), - prev_leaf[0]) + standalone=False), prev_leaf[0]) elif child.type == token.DEDENT: # Comment prefixes on DEDENT nodes also deserve special treatment, # because their final placement depends on their prefix. # We'll look for an ancestor of this child with a matching - # indentation, and insert the comment after it. - ancestor_at_indent = _FindAncestorAtIndent(child, prefix_indent) - if ancestor_at_indent.type == token.DEDENT: - comments = comment_prefix.split('\n') - - # lib2to3 places comments that should be separated into the same - # DEDENT node. For example, "comment 1" and "comment 2" will be - # combined. - # - # def _(): - # for x in y: - # pass - # # comment 1 - # - # # comment 2 - # pass - # - # In this case, we need to split them up ourselves. - before = [] - after = [] - after_lineno = comment_lineno - - index = 0 - while index < len(comments): - cmt = comments[index] - if not cmt.strip() or cmt.startswith(prefix_indent + '#'): - before.append(cmt) - else: - after_lineno += index - after.extend(comments[index:]) - break - index += 1 - - # Special case where the comment is inserted in the same - # indentation level as the DEDENT it was originally attached to. - pytree_utils.InsertNodesBefore( - _CreateCommentsFromPrefix( - '\n'.join(before) + '\n', - comment_lineno, - comment_column, - standalone=True), - ancestor_at_indent) - if after: - after_column = len(after[0]) - len(after[0].lstrip()) - comment_column -= comment_column - after_column - pytree_utils.InsertNodesAfter( - _CreateCommentsFromPrefix( - '\n'.join(after) + '\n', - after_lineno, - comment_column, - standalone=True), - _FindNextAncestor(ancestor_at_indent)) - else: - pytree_utils.InsertNodesAfter( + # indentation, and insert the comment before it if the ancestor is + # on a DEDENT node and after it otherwise. + # + # lib2to3 places comments that should be separated into the same + # DEDENT node. For example, "comment 1" and "comment 2" will be + # combined. + # + # def _(): + # for x in y: + # pass + # # comment 1 + # + # # comment 2 + # pass + # + # In this case, we need to split them up ourselves. + + # Split into groups of comments at decreasing levels of indentation + comment_groups = [] + comment_column = None + for cmt in comment_prefix.split('\n'): + col = cmt.find('#') + if col < 0: + if comment_column is None: + # Skip empty lines at the top of the first comment group + comment_lineno += 1 + continue + elif comment_column is None or col < comment_column: + comment_column = col + comment_indent = cmt[:comment_column] + comment_groups.append((comment_column, comment_indent, [])) + comment_groups[-1][-1].append(cmt) + + # Insert a node for each group + for comment_column, comment_indent, comment_group in comment_groups: + ancestor_at_indent = _FindAncestorAtIndent(child, comment_indent) + if ancestor_at_indent.type == token.DEDENT: + InsertNodes = pytree_utils.InsertNodesBefore # pylint: disable=invalid-name # noqa + else: + InsertNodes = pytree_utils.InsertNodesAfter # pylint: disable=invalid-name # noqa + InsertNodes( _CreateCommentsFromPrefix( - comment_prefix, + '\n'.join(comment_group) + '\n', comment_lineno, comment_column, - standalone=True), - ancestor_at_indent) + standalone=True), ancestor_at_indent) + comment_lineno += len(comment_group) else: # Otherwise there are two cases. # @@ -165,6 +153,16 @@ def _VisitNodeRec(node): # parent to insert into. See comments above # _STANDALONE_LINE_NODES for more details. node_with_line_parent = _FindNodeWithStandaloneLineParent(child) + + if pytree_utils.NodeName( + node_with_line_parent.parent) in {'funcdef', 'classdef'}: + # Keep a comment that's not attached to a function or class + # next to the object it is attached to. + comment_end = ( + comment_lineno + comment_prefix.rstrip('\n').count('\n')) + if comment_end < node_with_line_parent.lineno - 1: + node_with_line_parent = node_with_line_parent.parent + pytree_utils.InsertNodesBefore( _CreateCommentsFromPrefix( comment_prefix, comment_lineno, 0, standalone=True), @@ -174,21 +172,24 @@ def _VisitNodeRec(node): if comment_lineno == prev_leaf[0].lineno: comment_lines = comment_prefix.splitlines() value = comment_lines[0].lstrip() - comment_column = prev_leaf[0].column + len(prev_leaf[0].value) - comment_column += ( - len(comment_lines[0]) - len(comment_lines[0].lstrip())) - comment_leaf = pytree.Leaf( - type=token.COMMENT, - value=value.rstrip('\n'), - context=('', (comment_lineno, comment_column))) - pytree_utils.InsertNodesAfter([comment_leaf], prev_leaf[0]) - comment_prefix = '\n'.join(comment_lines[1:]) - comment_lineno += 1 + if value.rstrip('\n'): + comment_column = prev_leaf[0].column + comment_column += len(prev_leaf[0].value) + comment_column += ( + len(comment_lines[0]) - len(comment_lines[0].lstrip())) + comment_leaf = pytree.Leaf( + type=token.COMMENT, + value=value.rstrip('\n'), + context=('', (comment_lineno, comment_column))) + pytree_utils.InsertNodesAfter([comment_leaf], prev_leaf[0]) + comment_prefix = '\n'.join(comment_lines[1:]) + comment_lineno += 1 rindex = (0 if '\n' not in comment_prefix.rstrip() else comment_prefix.rstrip().rindex('\n') + 1) - comment_column = (len(comment_prefix[rindex:]) - - len(comment_prefix[rindex:].lstrip())) + comment_column = ( + len(comment_prefix[rindex:]) - + len(comment_prefix[rindex:].lstrip())) comments = _CreateCommentsFromPrefix( comment_prefix, comment_lineno, @@ -229,13 +230,13 @@ def _CreateCommentsFromPrefix(comment_prefix, while index < len(lines): comment_block = [] while index < len(lines) and lines[index].lstrip().startswith('#'): - comment_block.append(lines[index]) + comment_block.append(lines[index].strip()) index += 1 if comment_block: new_lineno = comment_lineno + index - 1 - comment_block[0] = comment_block[0].lstrip() - comment_block[-1] = comment_block[-1].rstrip('\n') + comment_block[0] = comment_block[0].strip() + comment_block[-1] = comment_block[-1].strip() comment_leaf = pytree.Leaf( type=token.COMMENT, value='\n'.join(comment_block), @@ -249,6 +250,7 @@ def _CreateCommentsFromPrefix(comment_prefix, return comments + # "Standalone line nodes" are tree nodes that have to start a new line in Python # code (and cannot follow a ';' or ':'). Other nodes, like 'expr_stmt', serve as # parents of other nodes but can come later in a line. This is a list of @@ -259,11 +261,11 @@ def _CreateCommentsFromPrefix(comment_prefix, # When splicing a standalone comment (i.e. a comment that appears on its own # line, not on the same line with other code), it's important to insert it into # an appropriate parent of the node it's attached to. An appropriate parent -# is the first "standaline line node" in the parent chain of a node. -_STANDALONE_LINE_NODES = frozenset(['suite', 'if_stmt', 'while_stmt', - 'for_stmt', 'try_stmt', 'with_stmt', - 'funcdef', 'classdef', 'decorated', - 'file_input']) +# is the first "standalone line node" in the parent chain of a node. +_STANDALONE_LINE_NODES = frozenset([ + 'suite', 'if_stmt', 'while_stmt', 'for_stmt', 'try_stmt', 'with_stmt', + 'funcdef', 'classdef', 'decorated', 'file_input' +]) def _FindNodeWithStandaloneLineParent(node): @@ -284,6 +286,7 @@ def _FindNodeWithStandaloneLineParent(node): # any pytree. return _FindNodeWithStandaloneLineParent(node.parent) + # "Statement nodes" are standalone statements. The don't have to start a new # line. _STATEMENT_NODES = frozenset(['simple_stmt']) | _STANDALONE_LINE_NODES @@ -335,16 +338,6 @@ def _FindAncestorAtIndent(node, indent): return _FindAncestorAtIndent(node.parent, indent) -def _FindNextAncestor(node): - if node.parent is None: - return node - - if node.parent.next_sibling is not None: - return node.parent.next_sibling - - return _FindNextAncestor(node.parent) - - def _AnnotateIndents(tree): """Annotate the tree with child_indent annotations. diff --git a/yapf/yapflib/continuation_splicer.py b/yapf/pytree/continuation_splicer.py similarity index 91% rename from yapf/yapflib/continuation_splicer.py rename to yapf/pytree/continuation_splicer.py index 0a459e17f..a8aef6606 100644 --- a/yapf/yapflib/continuation_splicer.py +++ b/yapf/pytree/continuation_splicer.py @@ -1,4 +1,4 @@ -# Copyright 2015-2016 Google Inc. All Rights Reserved. +# Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,10 +16,10 @@ The "backslash-newline" continuation marker is shoved into the node's prefix. Pull them out and make it into nodes of their own. - SpliceContinuations(): the main funciton exported by this module. + SpliceContinuations(): the main function exported by this module. """ -from lib2to3 import pytree +from yapf_third_party._ylib2to3 import pytree from yapf.yapflib import format_token diff --git a/yapf/yapflib/pytree_unwrapper.py b/yapf/pytree/pytree_unwrapper.py similarity index 60% rename from yapf/yapflib/pytree_unwrapper.py rename to yapf/pytree/pytree_unwrapper.py index f20d1e294..80e050fbd 100644 --- a/yapf/yapflib/pytree_unwrapper.py +++ b/yapf/pytree/pytree_unwrapper.py @@ -1,4 +1,4 @@ -# Copyright 2015-2016 Google Inc. All Rights Reserved. +# Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,13 +11,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""PyTreeUnwrapper - produces a list of unwrapped lines from a pytree. +"""PyTreeUnwrapper - produces a list of logical lines from a pytree. -[for a description of what an unwrapped line is, see unwrapped_line.py] +[for a description of what a logical line is, see logical_line.py] This is a pytree visitor that goes over a parse tree and produces a list of -UnwrappedLine containers from it, each with its own depth and containing all -the tokens that could fit on the line if there were no maximal line-length +LogicalLine containers from it, each with its own depth and containing all the +tokens that could fit on the line if there were no maximal line-length limitations. Note: a precondition to running this visitor and obtaining correct results is @@ -28,33 +28,43 @@ # The word "token" is overloaded within this module, so for clarity rename # the imported pgen2.token module. -from lib2to3 import pytree -from lib2to3.pgen2 import token as grammar_token +from yapf_third_party._ylib2to3 import pytree +from yapf_third_party._ylib2to3.pgen2 import token as grammar_token -from yapf.yapflib import pytree_utils -from yapf.yapflib import pytree_visitor -from yapf.yapflib import split_penalty -from yapf.yapflib import unwrapped_line +from yapf.pytree import pytree_utils +from yapf.pytree import pytree_visitor +from yapf.pytree import split_penalty +from yapf.yapflib import format_token +from yapf.yapflib import logical_line +from yapf.yapflib import object_state +from yapf.yapflib import style +from yapf.yapflib import subtypes + +_OPENING_BRACKETS = frozenset({'(', '[', '{'}) +_CLOSING_BRACKETS = frozenset({')', ']', '}'}) def UnwrapPyTree(tree): - """Create and return a list of unwrapped lines from the given pytree. + """Create and return a list of logical lines from the given pytree. Arguments: - tree: the top-level pytree node to unwrap. + tree: the top-level pytree node to unwrap.. Returns: - A list of UnwrappedLine objects. + A list of LogicalLine objects. """ unwrapper = PyTreeUnwrapper() unwrapper.Visit(tree) - uwlines = unwrapper.GetUnwrappedLines() - uwlines.sort(key=lambda x: x.lineno) - return uwlines + llines = unwrapper.GetLogicalLines() + llines.sort(key=lambda x: x.lineno) + return llines + # Grammar tokens considered as whitespace for the purpose of unwrapping. -_WHITESPACE_TOKENS = frozenset([grammar_token.NEWLINE, grammar_token.DEDENT, - grammar_token.INDENT, grammar_token.ENDMARKER]) +_WHITESPACE_TOKENS = frozenset([ + grammar_token.NEWLINE, grammar_token.DEDENT, grammar_token.INDENT, + grammar_token.ENDMARKER +]) class PyTreeUnwrapper(pytree_visitor.PyTreeVisitor): @@ -73,39 +83,40 @@ class PyTreeUnwrapper(pytree_visitor.PyTreeVisitor): """ def __init__(self): - # A list of all unwrapped lines finished visiting so far. - self._unwrapped_lines = [] + # A list of all logical lines finished visiting so far. + self._logical_lines = [] - # Builds up a "current" unwrapped line while visiting pytree nodes. Some - # nodes will finish a line and start a new one. - self._cur_unwrapped_line = unwrapped_line.UnwrappedLine(0) + # Builds up a "current" logical line while visiting pytree nodes. Some nodes + # will finish a line and start a new one. + self._cur_logical_line = logical_line.LogicalLine(0) # Current indentation depth. self._cur_depth = 0 - def GetUnwrappedLines(self): + def GetLogicalLines(self): """Fetch the result of the tree walk. Note: only call this after visiting the whole tree. Returns: - A list of UnwrappedLine objects. + A list of LogicalLine objects. """ # Make sure the last line that was being populated is flushed. self._StartNewLine() - return self._unwrapped_lines + return self._logical_lines def _StartNewLine(self): """Finish current line and start a new one. - Place the currently accumulated line into the _unwrapped_lines list and + Place the currently accumulated line into the _logical_lines list and start a new one. """ - if self._cur_unwrapped_line.tokens: - self._unwrapped_lines.append(self._cur_unwrapped_line) - _MatchBrackets(self._cur_unwrapped_line) - _AdjustSplitPenalty(self._cur_unwrapped_line) - self._cur_unwrapped_line = unwrapped_line.UnwrappedLine(self._cur_depth) + if self._cur_logical_line.tokens: + self._logical_lines.append(self._cur_logical_line) + _MatchBrackets(self._cur_logical_line) + _IdentifyParameterLists(self._cur_logical_line) + _AdjustSplitPenalty(self._cur_logical_line) + self._cur_logical_line = logical_line.LogicalLine(self._cur_depth) _STMT_TYPES = frozenset({ 'if_stmt', @@ -114,6 +125,8 @@ def _StartNewLine(self): 'try_stmt', 'expect_clause', 'with_stmt', + 'match_stmt', + 'case_block', 'funcdef', 'classdef', }) @@ -130,21 +143,23 @@ def Visit_simple_stmt(self, node): # standalone comment and in the case of it coming directly after the # funcdef, it is a "top" comment for the whole function. # TODO(eliben): add more relevant compound statements here. - single_stmt_suite = (node.parent and - pytree_utils.NodeName(node.parent) in self._STMT_TYPES) + single_stmt_suite = ( + node.parent and pytree_utils.NodeName(node.parent) in self._STMT_TYPES) is_comment_stmt = pytree_utils.IsCommentStatement(node) - if single_stmt_suite and not is_comment_stmt: + is_inside_match = node.parent and pytree_utils.NodeName( + node.parent) == 'match_stmt' + if (single_stmt_suite and not is_comment_stmt) or is_inside_match: self._cur_depth += 1 self._StartNewLine() self.DefaultNodeVisit(node) - if single_stmt_suite and not is_comment_stmt: + if (single_stmt_suite and not is_comment_stmt) or is_inside_match: self._cur_depth -= 1 def _VisitCompoundStatement(self, node, substatement_names): """Helper for visiting compound statements. Python compound statements serve as containers for other statements. Thus, - when we encounter a new compound statement we start a new unwrapped line. + when we encounter a new compound statement, we start a new logical line. Arguments: node: the node to visit. @@ -194,8 +209,13 @@ def Visit_funcdef(self, node): # pylint: disable=invalid-name def Visit_async_funcdef(self, node): # pylint: disable=invalid-name self._StartNewLine() - self.Visit(node.children[0]) - for child in node.children[1].children: + index = 0 + for child in node.children: + index += 1 + self.Visit(child) + if child.type == grammar_token.ASYNC: + break + for child in node.children[index].children: self.Visit(child) _CLASS_DEF_ELEMS = frozenset({'class'}) @@ -205,10 +225,23 @@ def Visit_classdef(self, node): # pylint: disable=invalid-name def Visit_async_stmt(self, node): # pylint: disable=invalid-name self._StartNewLine() - self.Visit(node.children[0]) - for child in node.children[1].children: + index = 0 + for child in node.children: + index += 1 + self.Visit(child) + if child.type == grammar_token.ASYNC: + break + for child in node.children[index].children: + if child.type == grammar_token.NAME and child.value == 'else': + self._StartNewLine() self.Visit(child) + def Visit_decorator(self, node): # pylint: disable=invalid-name + for child in node.children: + self.Visit(child) + if child.type == grammar_token.COMMENT and child == node.children[0]: + self._StartNewLine() + def Visit_decorators(self, node): # pylint: disable=invalid-name for child in node.children: self._StartNewLine() @@ -224,6 +257,20 @@ def Visit_decorated(self, node): # pylint: disable=invalid-name def Visit_with_stmt(self, node): # pylint: disable=invalid-name self._VisitCompoundStatement(node, self._WITH_STMT_ELEMS) + _MATCH_STMT_ELEMS = frozenset({'match', 'case'}) + + def Visit_match_stmt(self, node): # pylint: disable=invalid-name + self._VisitCompoundStatement(node, self._MATCH_STMT_ELEMS) + + # case_block refers to the grammar element name in Grammar.txt + _CASE_BLOCK_ELEMS = frozenset({'case'}) + + def Visit_case_block(self, node): + self._cur_depth += 1 + self._StartNewLine() + self._VisitCompoundStatement(node, self._CASE_BLOCK_ELEMS) + self._cur_depth -= 1 + def Visit_suite(self, node): # pylint: disable=invalid-name # A 'suite' starts a new indentation level in Python. self._cur_depth += 1 @@ -256,10 +303,14 @@ def Visit_typedargslist(self, node): # pylint: disable=invalid-name _DetermineMustSplitAnnotation(node) self.DefaultNodeVisit(node) + def Visit_subscriptlist(self, node): # pylint: disable=invalid-name + _DetermineMustSplitAnnotation(node) + self.DefaultNodeVisit(node) + def DefaultLeafVisit(self, leaf): """Default visitor for tree leaves. - A tree leaf is always just gets appended to the current unwrapped line. + A tree leaf is always just gets appended to the current logical line. Arguments: leaf: the leaf to visit. @@ -267,18 +318,15 @@ def DefaultLeafVisit(self, leaf): if leaf.type in _WHITESPACE_TOKENS: self._StartNewLine() elif leaf.type != grammar_token.COMMENT or leaf.value.strip(): - if leaf.value == ';': - # Split up multiple statements on one line. - self._StartNewLine() - else: - # Add non-whitespace tokens and comments that aren't empty. - self._cur_unwrapped_line.AppendNode(leaf) + # Add non-whitespace tokens and comments that aren't empty. + self._cur_logical_line.AppendToken( + format_token.FormatToken(leaf, pytree_utils.NodeName(leaf))) _BRACKET_MATCH = {')': '(', '}': '{', ']': '['} -def _MatchBrackets(uwline): +def _MatchBrackets(line): """Visit the node and match the brackets. For every open bracket ('[', '{', or '('), find the associated closing bracket @@ -286,47 +334,103 @@ def _MatchBrackets(uwline): or close bracket. Arguments: - uwline: (UnwrappedLine) An unwrapped line. + line: (LogicalLine) A logical line. """ bracket_stack = [] - for token in uwline.tokens: - if token.value in pytree_utils.OPENING_BRACKETS: + for token in line.tokens: + if token.value in _OPENING_BRACKETS: bracket_stack.append(token) - elif token.value in pytree_utils.CLOSING_BRACKETS: + elif token.value in _CLOSING_BRACKETS: bracket_stack[-1].matching_bracket = token token.matching_bracket = bracket_stack[-1] bracket_stack.pop() + for bracket in bracket_stack: + if id(pytree_utils.GetOpeningBracket(token.node)) == id(bracket.node): + bracket.container_elements.append(token) + token.container_opening = bracket + + +def _IdentifyParameterLists(line): + """Visit the node to create a state for parameter lists. + + For instance, a parameter is considered an "object" with its first and last + token uniquely identifying the object. -def _AdjustSplitPenalty(uwline): + Arguments: + line: (LogicalLine) A logical line. + """ + func_stack = [] + param_stack = [] + for tok in line.tokens: + # Identify parameter list objects. + if subtypes.FUNC_DEF in tok.subtypes: + assert tok.next_token.value == '(' + func_stack.append(tok.next_token) + continue + + if func_stack and tok.value == ')': + if tok == func_stack[-1].matching_bracket: + func_stack.pop() + continue + + # Identify parameter objects. + if subtypes.PARAMETER_START in tok.subtypes: + param_stack.append(tok) + + # Not "elif", a parameter could be a single token. + if param_stack and subtypes.PARAMETER_STOP in tok.subtypes: + start = param_stack.pop() + func_stack[-1].parameters.append(object_state.Parameter(start, tok)) + + +def _AdjustSplitPenalty(line): """Visit the node and adjust the split penalties if needed. A token shouldn't be split if it's not within a bracket pair. Mark any token that's not within a bracket pair as "unbreakable". Arguments: - uwline: (UnwrappedLine) An unwrapped line. + line: (LogicalLine) An logical line. """ bracket_level = 0 - for index, token in enumerate(uwline.tokens): + for index, token in enumerate(line.tokens): if index and not bracket_level: pytree_utils.SetNodeAnnotation(token.node, pytree_utils.Annotation.SPLIT_PENALTY, split_penalty.UNBREAKABLE) - if token.value in pytree_utils.OPENING_BRACKETS: + if token.value in _OPENING_BRACKETS: bracket_level += 1 - elif token.value in pytree_utils.CLOSING_BRACKETS: + elif token.value in _CLOSING_BRACKETS: bracket_level -= 1 def _DetermineMustSplitAnnotation(node): """Enforce a split in the list if the list ends with a comma.""" - if not _ContainsComments(node): + + def SplitBecauseTrailingComma(): + if style.Get('DISABLE_ENDING_COMMA_HEURISTIC'): + return False + token = next(node.parent.leaves()) + if token.value == '(': + if sum(1 for ch in node.children if ch.type == grammar_token.COMMA) < 2: + return False if (not isinstance(node.children[-1], pytree.Leaf) or node.children[-1].value != ','): - return + return False + return True + + def SplitBecauseListContainsComment(): + return (not style.Get('DISABLE_SPLIT_LIST_WITH_COMMENT') and + _ContainsComments(node)) + + if (not SplitBecauseTrailingComma() and + not SplitBecauseListContainsComment()): + return + num_children = len(node.children) index = 0 + _SetMustSplitOnFirstLeaf(node.children[0]) while index < num_children - 1: child = node.children[index] if isinstance(child, pytree.Leaf) and child.value == ',': @@ -351,11 +455,6 @@ def _ContainsComments(node): def _SetMustSplitOnFirstLeaf(node): """Set the "must split" annotation on the first leaf node.""" - - def FindFirstLeaf(node): - if isinstance(node, pytree.Leaf): - return node - return FindFirstLeaf(node.children[0]) - pytree_utils.SetNodeAnnotation( - FindFirstLeaf(node), pytree_utils.Annotation.MUST_SPLIT, True) + pytree_utils.FirstLeafNode(node), pytree_utils.Annotation.MUST_SPLIT, + True) diff --git a/yapf/yapflib/pytree_utils.py b/yapf/pytree/pytree_utils.py similarity index 80% rename from yapf/yapflib/pytree_utils.py rename to yapf/pytree/pytree_utils.py index 9fea5e69b..e7aa6f59c 100644 --- a/yapf/yapflib/pytree_utils.py +++ b/yapf/pytree/pytree_utils.py @@ -1,4 +1,4 @@ -# Copyright 2015-2016 Google Inc. All Rights Reserved. +# Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -25,11 +25,13 @@ """ import ast -from lib2to3 import pygram -from lib2to3 import pytree -from lib2to3.pgen2 import driver -from lib2to3.pgen2 import parse -from lib2to3.pgen2 import token +import os + +from yapf_third_party._ylib2to3 import pygram +from yapf_third_party._ylib2to3 import pytree +from yapf_third_party._ylib2to3.pgen2 import driver +from yapf_third_party._ylib2to3.pgen2 import parse +from yapf_third_party._ylib2to3.pgen2 import token # TODO(eliben): We may want to get rid of this filtering at some point once we # have a better understanding of what information we need from the tree. Then, @@ -37,9 +39,6 @@ # unwrapper. NONSEMANTIC_TOKENS = frozenset(['DEDENT', 'INDENT', 'NEWLINE', 'ENDMARKER']) -OPENING_BRACKETS = frozenset({'(', '[', '{'}) -CLOSING_BRACKETS = frozenset({')', ']', '}'}) - class Annotation(object): """Annotation names associated with pytrees.""" @@ -67,16 +66,28 @@ def NodeName(node): else: return pygram.python_grammar.number2symbol[node.type] + +def FirstLeafNode(node): + if isinstance(node, pytree.Leaf): + return node + return FirstLeafNode(node.children[0]) + + +def LastLeafNode(node): + if isinstance(node, pytree.Leaf): + return node + return LastLeafNode(node.children[-1]) + + # lib2to3 thoughtfully provides pygram.python_grammar_no_print_statement for # parsing Python 3 code that wouldn't parse otherwise (when 'print' is used in a # context where a keyword is disallowed). # It forgets to do the same for 'exec' though. Luckily, Python is amenable to # monkey-patching. -_GRAMMAR_FOR_PY3 = pygram.python_grammar_no_print_statement.copy() -del _GRAMMAR_FOR_PY3.keywords['exec'] - -_GRAMMAR_FOR_PY2 = pygram.python_grammar.copy() -del _GRAMMAR_FOR_PY2.keywords['nonlocal'] +# Note that pygram.python_grammar_no_print_and_exec_statement with "_and_exec" +# will require Python >=3.8. +_PYTHON_GRAMMAR = pygram.python_grammar_no_print_statement.copy() +del _PYTHON_GRAMMAR.keywords['exec'] def ParseCodeToTree(code): @@ -94,25 +105,16 @@ def ParseCodeToTree(code): """ # This function is tiny, but the incantation for invoking the parser correctly # is sufficiently magical to be worth abstracting away. + if not code.endswith(os.linesep): + code += os.linesep + try: - # Try to parse using a Python 3 grammar, which is more permissive (print and - # exec are not keywords). - parser_driver = driver.Driver(_GRAMMAR_FOR_PY3, convert=pytree.convert) + parser_driver = driver.Driver(_PYTHON_GRAMMAR, convert=pytree.convert) tree = parser_driver.parse_string(code, debug=False) except parse.ParseError: - # Now try to parse using a Python 2 grammar; If this fails, then - # there's something else wrong with the code. - try: - parser_driver = driver.Driver(_GRAMMAR_FOR_PY2, convert=pytree.convert) - tree = parser_driver.parse_string(code, debug=False) - except parse.ParseError: - # Raise a syntax error if the code is invalid python syntax. - try: - ast.parse(code) - except SyntaxError as e: - raise e - else: - raise + # Raise a syntax error if the code is invalid python syntax. + ast.parse(code) + raise return _WrapEndMarker(tree) @@ -197,6 +199,7 @@ def _InsertNodeAt(new_node, target, after=False): raise RuntimeError('unable to find insertion point for target node', (target,)) + # The following constant and functions implement a simple custom annotation # mechanism for pytree nodes. We attach new attributes to nodes. Each attribute # is prefixed with _NODE_ANNOTATION_PREFIX. These annotations should only be @@ -204,6 +207,18 @@ def _InsertNodeAt(new_node, target, after=False): _NODE_ANNOTATION_PREFIX = '_yapf_annotation_' +def CopyYapfAnnotations(src, dst): + """Copy all YAPF annotations from the source node to the destination node. + + Arguments: + src: the source node. + dst: the destination node. + """ + for annotation in dir(src): + if annotation.startswith(_NODE_ANNOTATION_PREFIX): + setattr(dst, annotation, getattr(src, annotation, None)) + + def GetNodeAnnotation(node, annotation, default=None): """Get annotation value from a node. @@ -256,6 +271,28 @@ def RemoveSubtypeAnnotation(node, value): SetNodeAnnotation(node, Annotation.SUBTYPE, attr) +def GetOpeningBracket(node): + """Get opening bracket value from a node. + + Arguments: + node: the node. + + Returns: + The opening bracket node or None if it couldn't find one. + """ + return getattr(node, _NODE_ANNOTATION_PREFIX + 'container_bracket', None) + + +def SetOpeningBracket(node, bracket): + """Set opening bracket value for a node. + + Arguments: + node: the node. + bracket: opening bracket to set. + """ + setattr(node, _NODE_ANNOTATION_PREFIX + 'container_bracket', bracket) + + def DumpNodeToString(node): """Dump a string representation of the given node. For debugging. @@ -266,13 +303,15 @@ def DumpNodeToString(node): The string representation. """ if isinstance(node, pytree.Leaf): - fmt = '{name}({value}) [lineno={lineno}, column={column}, prefix={prefix}]' + fmt = ('{name}({value}) [lineno={lineno}, column={column}, ' + 'prefix={prefix}, penalty={penalty}]') return fmt.format( name=NodeName(node), value=_PytreeNodeRepr(node), lineno=node.lineno, column=node.column, - prefix=repr(node.prefix)) + prefix=repr(node.prefix), + penalty=GetNodeAnnotation(node, Annotation.SPLIT_PENALTY, None)) else: fmt = '{node} [{len} children] [child_indent="{indent}"]' return fmt.format( diff --git a/yapf/yapflib/pytree_visitor.py b/yapf/pytree/pytree_visitor.py similarity index 95% rename from yapf/yapflib/pytree_visitor.py rename to yapf/pytree/pytree_visitor.py index c13aea5c4..ec2cdb7e8 100644 --- a/yapf/yapflib/pytree_visitor.py +++ b/yapf/pytree/pytree_visitor.py @@ -1,4 +1,4 @@ -# Copyright 2015-2016 Google Inc. All Rights Reserved. +# Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,16 +19,16 @@ It also exports a basic "dumping" visitor that dumps a textual representation of a pytree into a stream. - PyTreeVisitor: a generic visitor pattern fo pytrees. + PyTreeVisitor: a generic visitor pattern for pytrees. PyTreeDumper: a configurable "dumper" for displaying pytrees. DumpPyTree(): a convenience function to dump a pytree. """ import sys -from lib2to3 import pytree +from yapf_third_party._ylib2to3 import pytree -from yapf.yapflib import pytree_utils +from yapf.pytree import pytree_utils class PyTreeVisitor(object): diff --git a/yapf/pytree/split_penalty.py b/yapf/pytree/split_penalty.py new file mode 100644 index 000000000..03b36384a --- /dev/null +++ b/yapf/pytree/split_penalty.py @@ -0,0 +1,632 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Computation of split penalties before/between tokens.""" + +import re + +from yapf_third_party._ylib2to3 import pytree +from yapf_third_party._ylib2to3.pgen2 import token as grammar_token + +from yapf.pytree import pytree_utils +from yapf.pytree import pytree_visitor +from yapf.yapflib import style +from yapf.yapflib import subtypes + +# TODO(morbo): Document the annotations in a centralized place. E.g., the +# README file. +UNBREAKABLE = 1000 * 1000 +NAMED_ASSIGN = 15000 +DOTTED_NAME = 4000 +VERY_STRONGLY_CONNECTED = 3500 +STRONGLY_CONNECTED = 3000 +CONNECTED = 500 +TOGETHER = 100 + +OR_TEST = 1000 +AND_TEST = 1100 +NOT_TEST = 1200 +COMPARISON = 1300 +STAR_EXPR = 1300 +EXPR = 1400 +XOR_EXPR = 1500 +AND_EXPR = 1700 +SHIFT_EXPR = 1800 +ARITH_EXPR = 1900 +TERM = 2000 +FACTOR = 2100 +POWER = 2200 +ATOM = 2300 +ONE_ELEMENT_ARGUMENT = 500 +SUBSCRIPT = 6000 + + +def ComputeSplitPenalties(tree): + """Compute split penalties on tokens in the given parse tree. + + Arguments: + tree: the top-level pytree node to annotate with penalties. + """ + _SplitPenaltyAssigner().Visit(tree) + + +class _SplitPenaltyAssigner(pytree_visitor.PyTreeVisitor): + """Assigns split penalties to tokens, based on parse tree structure. + + Split penalties are attached as annotations to tokens. + """ + + def Visit(self, node): + if not hasattr(node, 'is_pseudo'): # Ignore pseudo tokens. + super(_SplitPenaltyAssigner, self).Visit(node) + + def Visit_import_as_names(self, node): # pyline: disable=invalid-name + # import_as_names ::= import_as_name (',' import_as_name)* [','] + self.DefaultNodeVisit(node) + prev_child = None + for child in node.children: + if (prev_child and isinstance(prev_child, pytree.Leaf) and + prev_child.value == ','): + _SetSplitPenalty(child, style.Get('SPLIT_PENALTY_IMPORT_NAMES')) + prev_child = child + + def Visit_classdef(self, node): # pylint: disable=invalid-name + # classdef ::= 'class' NAME ['(' [arglist] ')'] ':' suite + # + # NAME + _SetUnbreakable(node.children[1]) + if len(node.children) > 4: + # opening '(' + _SetUnbreakable(node.children[2]) + # ':' + _SetUnbreakable(node.children[-2]) + self.DefaultNodeVisit(node) + + def Visit_funcdef(self, node): # pylint: disable=invalid-name + # funcdef ::= 'def' NAME parameters ['->' test] ':' suite + # + # Can't break before the function name and before the colon. The parameters + # are handled by child iteration. + colon_idx = 1 + while pytree_utils.NodeName(node.children[colon_idx]) == 'simple_stmt': + colon_idx += 1 + _SetUnbreakable(node.children[colon_idx]) + arrow_idx = -1 + while colon_idx < len(node.children): + if isinstance(node.children[colon_idx], pytree.Leaf): + if node.children[colon_idx].value == ':': + break + if node.children[colon_idx].value == '->': + arrow_idx = colon_idx + colon_idx += 1 + _SetUnbreakable(node.children[colon_idx]) + self.DefaultNodeVisit(node) + if arrow_idx > 0: + _SetSplitPenalty( + pytree_utils.LastLeafNode(node.children[arrow_idx - 1]), 0) + _SetUnbreakable(node.children[arrow_idx]) + _SetStronglyConnected(node.children[arrow_idx + 1]) + + def Visit_lambdef(self, node): # pylint: disable=invalid-name + # lambdef ::= 'lambda' [varargslist] ':' test + # Loop over the lambda up to and including the colon. + allow_multiline_lambdas = style.Get('ALLOW_MULTILINE_LAMBDAS') + if not allow_multiline_lambdas: + for child in node.children: + if child.type == grammar_token.COMMENT: + if re.search(r'pylint:.*disable=.*\bg-long-lambda', child.value): + allow_multiline_lambdas = True + break + + if allow_multiline_lambdas: + _SetExpressionPenalty(node, STRONGLY_CONNECTED) + else: + _SetExpressionPenalty(node, VERY_STRONGLY_CONNECTED) + + def Visit_parameters(self, node): # pylint: disable=invalid-name + # parameters ::= '(' [typedargslist] ')' + self.DefaultNodeVisit(node) + + # Can't break before the opening paren of a parameter list. + _SetUnbreakable(node.children[0]) + if not (style.Get('INDENT_CLOSING_BRACKETS') or + style.Get('DEDENT_CLOSING_BRACKETS')): + _SetStronglyConnected(node.children[-1]) + + def Visit_arglist(self, node): # pylint: disable=invalid-name + # arglist ::= argument (',' argument)* [','] + if node.children[0].type == grammar_token.STAR: + # Python 3 treats a star expression as a specific expression type. + # Process it in that method. + self.Visit_star_expr(node) + return + + self.DefaultNodeVisit(node) + + for index in range(1, len(node.children)): + child = node.children[index] + if isinstance(child, pytree.Leaf) and child.value == ',': + _SetUnbreakable(child) + + for child in node.children: + if pytree_utils.NodeName(child) == 'atom': + _IncreasePenalty(child, CONNECTED) + + def Visit_argument(self, node): # pylint: disable=invalid-name + # argument ::= test [comp_for] | test '=' test # Really [keyword '='] test + self.DefaultNodeVisit(node) + + for index in range(1, len(node.children) - 1): + child = node.children[index] + if isinstance(child, pytree.Leaf) and child.value == '=': + _SetSplitPenalty( + pytree_utils.FirstLeafNode(node.children[index]), NAMED_ASSIGN) + _SetSplitPenalty( + pytree_utils.FirstLeafNode(node.children[index + 1]), NAMED_ASSIGN) + + def Visit_tname(self, node): # pylint: disable=invalid-name + # tname ::= NAME [':' test] + self.DefaultNodeVisit(node) + + for index in range(1, len(node.children) - 1): + child = node.children[index] + if isinstance(child, pytree.Leaf) and child.value == ':': + _SetSplitPenalty( + pytree_utils.FirstLeafNode(node.children[index]), NAMED_ASSIGN) + _SetSplitPenalty( + pytree_utils.FirstLeafNode(node.children[index + 1]), NAMED_ASSIGN) + + def Visit_dotted_name(self, node): # pylint: disable=invalid-name + # dotted_name ::= NAME ('.' NAME)* + for child in node.children: + self.Visit(child) + start = 2 if hasattr(node.children[0], 'is_pseudo') else 1 + for i in range(start, len(node.children)): + _SetUnbreakable(node.children[i]) + + def Visit_dictsetmaker(self, node): # pylint: disable=invalid-name + # dictsetmaker ::= ( (test ':' test + # (comp_for | (',' test ':' test)* [','])) | + # (test (comp_for | (',' test)* [','])) ) + for child in node.children: + self.Visit(child) + if child.type == grammar_token.COLON: + # This is a key to a dictionary. We don't want to split the key if at + # all possible. + _SetStronglyConnected(child) + + def Visit_trailer(self, node): # pylint: disable=invalid-name + # trailer ::= '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME + if node.children[0].value == '.': + before = style.Get('SPLIT_BEFORE_DOT') + _SetSplitPenalty(node.children[0], + VERY_STRONGLY_CONNECTED if before else DOTTED_NAME) + _SetSplitPenalty(node.children[1], + DOTTED_NAME if before else VERY_STRONGLY_CONNECTED) + elif len(node.children) == 2: + # Don't split an empty argument list if at all possible. + _SetSplitPenalty(node.children[1], VERY_STRONGLY_CONNECTED) + elif len(node.children) == 3: + name = pytree_utils.NodeName(node.children[1]) + if name in {'argument', 'comparison'}: + # Don't split an argument list with one element if at all possible. + _SetStronglyConnected(node.children[1]) + if (len(node.children[1].children) > 1 and + pytree_utils.NodeName(node.children[1].children[1]) == 'comp_for'): + # Don't penalize splitting before a comp_for expression. + _SetSplitPenalty(pytree_utils.FirstLeafNode(node.children[1]), 0) + else: + _SetSplitPenalty( + pytree_utils.FirstLeafNode(node.children[1]), + ONE_ELEMENT_ARGUMENT) + elif (node.children[0].type == grammar_token.LSQB and + len(node.children[1].children) > 2 and + (name.endswith('_test') or name.endswith('_expr'))): + _SetStronglyConnected(node.children[1].children[0]) + _SetStronglyConnected(node.children[1].children[2]) + + # Still allow splitting around the operator. + split_before = ((name.endswith('_test') and + style.Get('SPLIT_BEFORE_LOGICAL_OPERATOR')) or + (name.endswith('_expr') and + style.Get('SPLIT_BEFORE_BITWISE_OPERATOR'))) + if split_before: + _SetSplitPenalty( + pytree_utils.LastLeafNode(node.children[1].children[1]), 0) + else: + _SetSplitPenalty( + pytree_utils.FirstLeafNode(node.children[1].children[2]), 0) + + # Don't split the ending bracket of a subscript list. + _RecAnnotate(node.children[-1], pytree_utils.Annotation.SPLIT_PENALTY, + VERY_STRONGLY_CONNECTED) + elif name not in { + 'arglist', 'argument', 'term', 'or_test', 'and_test', 'comparison', + 'atom', 'power' + }: + # Don't split an argument list with one element if at all possible. + stypes = pytree_utils.GetNodeAnnotation( + pytree_utils.FirstLeafNode(node), pytree_utils.Annotation.SUBTYPE) + if stypes and subtypes.SUBSCRIPT_BRACKET in stypes: + _IncreasePenalty(node, SUBSCRIPT) + + # Bump up the split penalty for the first part of a subscript. We + # would rather not split there. + _IncreasePenalty(node.children[1], CONNECTED) + else: + _SetStronglyConnected(node.children[1], node.children[2]) + + if name == 'arglist': + _SetStronglyConnected(node.children[-1]) + + self.DefaultNodeVisit(node) + + def Visit_power(self, node): # pylint: disable=invalid-name,missing-docstring + # power ::= atom trailer* ['**' factor] + self.DefaultNodeVisit(node) + + # When atom is followed by a trailer, we can not break between them. + # E.g. arr[idx] - no break allowed between 'arr' and '['. + if (len(node.children) > 1 and + pytree_utils.NodeName(node.children[1]) == 'trailer'): + # children[1] itself is a whole trailer: we don't want to + # mark all of it as unbreakable, only its first token: (, [ or . + first = pytree_utils.FirstLeafNode(node.children[1]) + if first.value != '.': + _SetUnbreakable(node.children[1].children[0]) + + # A special case when there are more trailers in the sequence. Given: + # atom tr1 tr2 + # The last token of tr1 and the first token of tr2 comprise an unbreakable + # region. For example: foo.bar.baz(1) + # We can't put breaks between either of the '.', '(', or '[' and the names + # *preceding* them. + prev_trailer_idx = 1 + while prev_trailer_idx < len(node.children) - 1: + cur_trailer_idx = prev_trailer_idx + 1 + cur_trailer = node.children[cur_trailer_idx] + if pytree_utils.NodeName(cur_trailer) != 'trailer': + break + + # Now we know we have two trailers one after the other + prev_trailer = node.children[prev_trailer_idx] + if prev_trailer.children[-1].value != ')': + # Set the previous node unbreakable if it's not a function call: + # atom tr1() tr2 + # It may be necessary (though undesirable) to split up a previous + # function call's parentheses to the next line. + _SetStronglyConnected(prev_trailer.children[-1]) + _SetStronglyConnected(cur_trailer.children[0]) + prev_trailer_idx = cur_trailer_idx + + # We don't want to split before the last ')' of a function call. This also + # takes care of the special case of: + # atom tr1 tr2 ... trn + # where the 'tr#' are trailers that may end in a ')'. + for trailer in node.children[1:]: + if pytree_utils.NodeName(trailer) != 'trailer': + break + if trailer.children[0].value in '([': + if len(trailer.children) > 2: + stypes = pytree_utils.GetNodeAnnotation( + trailer.children[0], pytree_utils.Annotation.SUBTYPE) + if stypes and subtypes.SUBSCRIPT_BRACKET in stypes: + _SetStronglyConnected( + pytree_utils.FirstLeafNode(trailer.children[1])) + + last_child_node = pytree_utils.LastLeafNode(trailer) + if last_child_node.value.strip().startswith('#'): + last_child_node = last_child_node.prev_sibling + if not (style.Get('INDENT_CLOSING_BRACKETS') or + style.Get('DEDENT_CLOSING_BRACKETS')): + last = pytree_utils.LastLeafNode(last_child_node.prev_sibling) + if last.value != ',': + if last_child_node.value == ']': + _SetUnbreakable(last_child_node) + else: + _SetSplitPenalty(last_child_node, VERY_STRONGLY_CONNECTED) + else: + # If the trailer's children are '()', then make it a strongly + # connected region. It's sometimes necessary, though undesirable, to + # split the two. + _SetStronglyConnected(trailer.children[-1]) + + def Visit_subscriptlist(self, node): # pylint: disable=invalid-name + # subscriptlist ::= subscript (',' subscript)* [','] + self.DefaultNodeVisit(node) + _SetSplitPenalty(pytree_utils.FirstLeafNode(node), 0) + prev_child = None + for child in node.children: + if prev_child and prev_child.type == grammar_token.COMMA: + _SetSplitPenalty(pytree_utils.FirstLeafNode(child), 0) + prev_child = child + + def Visit_subscript(self, node): # pylint: disable=invalid-name + # subscript ::= test | [test] ':' [test] [sliceop] + _SetStronglyConnected(*node.children) + self.DefaultNodeVisit(node) + + def Visit_comp_for(self, node): # pylint: disable=invalid-name + # comp_for ::= 'for' exprlist 'in' testlist_safe [comp_iter] + _SetSplitPenalty(pytree_utils.FirstLeafNode(node), 0) + _SetStronglyConnected(*node.children[1:]) + self.DefaultNodeVisit(node) + + def Visit_old_comp_for(self, node): # pylint: disable=invalid-name + # Python 3.7 + self.Visit_comp_for(node) + + def Visit_comp_if(self, node): # pylint: disable=invalid-name + # comp_if ::= 'if' old_test [comp_iter] + _SetSplitPenalty(node.children[0], + style.Get('SPLIT_PENALTY_BEFORE_IF_EXPR')) + _SetStronglyConnected(*node.children[1:]) + self.DefaultNodeVisit(node) + + def Visit_old_comp_if(self, node): # pylint: disable=invalid-name + # Python 3.7 + self.Visit_comp_if(node) + + def Visit_test(self, node): # pylint: disable=invalid-name + # test ::= or_test ['if' or_test 'else' test] | lambdef + _IncreasePenalty(node, OR_TEST) + self.DefaultNodeVisit(node) + + def Visit_or_test(self, node): # pylint: disable=invalid-name + # or_test ::= and_test ('or' and_test)* + self.DefaultNodeVisit(node) + _IncreasePenalty(node, OR_TEST) + index = 1 + while index + 1 < len(node.children): + if style.Get('SPLIT_BEFORE_LOGICAL_OPERATOR'): + _DecrementSplitPenalty( + pytree_utils.FirstLeafNode(node.children[index]), OR_TEST) + else: + _DecrementSplitPenalty( + pytree_utils.FirstLeafNode(node.children[index + 1]), OR_TEST) + index += 2 + + def Visit_and_test(self, node): # pylint: disable=invalid-name + # and_test ::= not_test ('and' not_test)* + self.DefaultNodeVisit(node) + _IncreasePenalty(node, AND_TEST) + index = 1 + while index + 1 < len(node.children): + if style.Get('SPLIT_BEFORE_LOGICAL_OPERATOR'): + _DecrementSplitPenalty( + pytree_utils.FirstLeafNode(node.children[index]), AND_TEST) + else: + _DecrementSplitPenalty( + pytree_utils.FirstLeafNode(node.children[index + 1]), AND_TEST) + index += 2 + + def Visit_not_test(self, node): # pylint: disable=invalid-name + # not_test ::= 'not' not_test | comparison + self.DefaultNodeVisit(node) + _IncreasePenalty(node, NOT_TEST) + + def Visit_comparison(self, node): # pylint: disable=invalid-name + # comparison ::= expr (comp_op expr)* + self.DefaultNodeVisit(node) + if len(node.children) == 3 and _StronglyConnectedCompOp(node): + _IncreasePenalty(node.children[1], VERY_STRONGLY_CONNECTED) + _SetSplitPenalty( + pytree_utils.FirstLeafNode(node.children[2]), STRONGLY_CONNECTED) + else: + _IncreasePenalty(node, COMPARISON) + + def Visit_star_expr(self, node): # pylint: disable=invalid-name + # star_expr ::= '*' expr + self.DefaultNodeVisit(node) + _IncreasePenalty(node, STAR_EXPR) + + def Visit_expr(self, node): # pylint: disable=invalid-name + # expr ::= xor_expr ('|' xor_expr)* + self.DefaultNodeVisit(node) + _IncreasePenalty(node, EXPR) + _SetBitwiseOperandPenalty(node, '|') + + def Visit_xor_expr(self, node): # pylint: disable=invalid-name + # xor_expr ::= and_expr ('^' and_expr)* + self.DefaultNodeVisit(node) + _IncreasePenalty(node, XOR_EXPR) + _SetBitwiseOperandPenalty(node, '^') + + def Visit_and_expr(self, node): # pylint: disable=invalid-name + # and_expr ::= shift_expr ('&' shift_expr)* + self.DefaultNodeVisit(node) + _IncreasePenalty(node, AND_EXPR) + _SetBitwiseOperandPenalty(node, '&') + + def Visit_shift_expr(self, node): # pylint: disable=invalid-name + # shift_expr ::= arith_expr (('<<'|'>>') arith_expr)* + self.DefaultNodeVisit(node) + _IncreasePenalty(node, SHIFT_EXPR) + + _ARITH_OPS = frozenset({'PLUS', 'MINUS'}) + + def Visit_arith_expr(self, node): # pylint: disable=invalid-name + # arith_expr ::= term (('+'|'-') term)* + self.DefaultNodeVisit(node) + _IncreasePenalty(node, ARITH_EXPR) + _SetExpressionOperandPenalty(node, self._ARITH_OPS) + + _TERM_OPS = frozenset({'STAR', 'AT', 'SLASH', 'PERCENT', 'DOUBLESLASH'}) + + def Visit_term(self, node): # pylint: disable=invalid-name + # term ::= factor (('*'|'@'|'/'|'%'|'//') factor)* + self.DefaultNodeVisit(node) + _IncreasePenalty(node, TERM) + _SetExpressionOperandPenalty(node, self._TERM_OPS) + + def Visit_factor(self, node): # pyline: disable=invalid-name + # factor ::= ('+'|'-'|'~') factor | power + self.DefaultNodeVisit(node) + _IncreasePenalty(node, FACTOR) + + def Visit_atom(self, node): # pylint: disable=invalid-name + # atom ::= ('(' [yield_expr|testlist_gexp] ')' + # '[' [listmaker] ']' | + # '{' [dictsetmaker] '}') + self.DefaultNodeVisit(node) + if (node.children[0].value == '(' and + not hasattr(node.children[0], 'is_pseudo')): + if node.children[-1].value == ')': + if pytree_utils.NodeName(node.parent) == 'if_stmt': + _SetSplitPenalty(node.children[-1], STRONGLY_CONNECTED) + else: + if len(node.children) > 2: + _SetSplitPenalty(pytree_utils.FirstLeafNode(node.children[1]), EXPR) + _SetSplitPenalty(node.children[-1], ATOM) + elif node.children[0].value in '[{' and len(node.children) == 2: + # Keep empty containers together if we can. + _SetUnbreakable(node.children[-1]) + + def Visit_testlist_gexp(self, node): # pylint: disable=invalid-name + self.DefaultNodeVisit(node) + prev_was_comma = False + for child in node.children: + if isinstance(child, pytree.Leaf) and child.value == ',': + _SetUnbreakable(child) + prev_was_comma = True + else: + if prev_was_comma: + _SetSplitPenalty(pytree_utils.FirstLeafNode(child), TOGETHER) + prev_was_comma = False + + +def _SetUnbreakable(node): + """Set an UNBREAKABLE penalty annotation for the given node.""" + _RecAnnotate(node, pytree_utils.Annotation.SPLIT_PENALTY, UNBREAKABLE) + + +def _SetStronglyConnected(*nodes): + """Set a STRONGLY_CONNECTED penalty annotation for the given nodes.""" + for node in nodes: + _RecAnnotate(node, pytree_utils.Annotation.SPLIT_PENALTY, + STRONGLY_CONNECTED) + + +def _SetExpressionPenalty(node, penalty): + """Set a penalty annotation on children nodes.""" + + def RecExpression(node, first_child_leaf): + if node is first_child_leaf: + return + + if isinstance(node, pytree.Leaf): + if node.value in {'(', 'for', 'if'}: + return + penalty_annotation = pytree_utils.GetNodeAnnotation( + node, pytree_utils.Annotation.SPLIT_PENALTY, default=0) + if penalty_annotation < penalty: + _SetSplitPenalty(node, penalty) + else: + for child in node.children: + RecExpression(child, first_child_leaf) + + RecExpression(node, pytree_utils.FirstLeafNode(node)) + + +def _SetBitwiseOperandPenalty(node, op): + for index in range(1, len(node.children) - 1): + child = node.children[index] + if isinstance(child, pytree.Leaf) and child.value == op: + if style.Get('SPLIT_BEFORE_BITWISE_OPERATOR'): + _SetSplitPenalty(child, style.Get('SPLIT_PENALTY_BITWISE_OPERATOR')) + else: + _SetSplitPenalty( + pytree_utils.FirstLeafNode(node.children[index + 1]), + style.Get('SPLIT_PENALTY_BITWISE_OPERATOR')) + + +def _SetExpressionOperandPenalty(node, ops): + for index in range(1, len(node.children) - 1): + child = node.children[index] + if pytree_utils.NodeName(child) in ops: + if style.Get('SPLIT_BEFORE_ARITHMETIC_OPERATOR'): + _SetSplitPenalty(child, style.Get('SPLIT_PENALTY_ARITHMETIC_OPERATOR')) + else: + _SetSplitPenalty( + pytree_utils.FirstLeafNode(node.children[index + 1]), + style.Get('SPLIT_PENALTY_ARITHMETIC_OPERATOR')) + + +def _IncreasePenalty(node, amt): + """Increase a penalty annotation on children nodes.""" + + def RecExpression(node, first_child_leaf): + if node is first_child_leaf: + return + + if isinstance(node, pytree.Leaf): + if node.value in {'(', 'for'}: + return + penalty = pytree_utils.GetNodeAnnotation( + node, pytree_utils.Annotation.SPLIT_PENALTY, default=0) + _SetSplitPenalty(node, penalty + amt) + else: + for child in node.children: + RecExpression(child, first_child_leaf) + + RecExpression(node, pytree_utils.FirstLeafNode(node)) + + +def _RecAnnotate(tree, annotate_name, annotate_value): + """Recursively set the given annotation on all leafs of the subtree. + + Takes care to only increase the penalty. If the node already has a higher + or equal penalty associated with it, this is a no-op. + + Args: + tree: subtree to annotate + annotate_name: name of the annotation to set + annotate_value: value of the annotation to set + """ + for child in tree.children: + _RecAnnotate(child, annotate_name, annotate_value) + if isinstance(tree, pytree.Leaf): + cur_annotate = pytree_utils.GetNodeAnnotation( + tree, annotate_name, default=0) + if cur_annotate < annotate_value: + pytree_utils.SetNodeAnnotation(tree, annotate_name, annotate_value) + + +_COMP_OPS = frozenset({'==', '!=', '<=', '<', '>', '>=', '<>', 'in', 'is'}) + + +def _StronglyConnectedCompOp(op): + if (len(op.children[1].children) == 2 and + pytree_utils.NodeName(op.children[1]) == 'comp_op'): + if (pytree_utils.FirstLeafNode(op.children[1]).value == 'not' and + pytree_utils.LastLeafNode(op.children[1]).value == 'in'): + return True + if (pytree_utils.FirstLeafNode(op.children[1]).value == 'is' and + pytree_utils.LastLeafNode(op.children[1]).value == 'not'): + return True + if (isinstance(op.children[1], pytree.Leaf) and + op.children[1].value in _COMP_OPS): + return True + return False + + +def _DecrementSplitPenalty(node, amt): + penalty = pytree_utils.GetNodeAnnotation( + node, pytree_utils.Annotation.SPLIT_PENALTY, default=amt) + penalty = penalty - amt if amt < penalty else 0 + _SetSplitPenalty(node, penalty) + + +def _SetSplitPenalty(node, penalty): + pytree_utils.SetNodeAnnotation(node, pytree_utils.Annotation.SPLIT_PENALTY, + penalty) diff --git a/yapf/pytree/subtype_assigner.py b/yapf/pytree/subtype_assigner.py new file mode 100644 index 000000000..e3b32777a --- /dev/null +++ b/yapf/pytree/subtype_assigner.py @@ -0,0 +1,508 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Subtype assigner for format tokens. + +This module assigns extra type information to format tokens. This information is +more specific than whether something is an operator or an identifier. For +instance, it can specify if a node in the tree is part of a subscript. + + AssignSubtypes(): the main function exported by this module. + +Annotations: + subtype: The subtype of a pytree token. See 'subtypes' module for a list of + subtypes. +""" + +from yapf_third_party._ylib2to3 import pytree +from yapf_third_party._ylib2to3.pgen2 import token as grammar_token +from yapf_third_party._ylib2to3.pygram import python_symbols as syms + +from yapf.pytree import pytree_utils +from yapf.pytree import pytree_visitor +from yapf.yapflib import style +from yapf.yapflib import subtypes + + +def AssignSubtypes(tree): + """Run the subtype assigner visitor over the tree, modifying it in place. + + Arguments: + tree: the top-level pytree node to annotate with subtypes. + """ + subtype_assigner = _SubtypeAssigner() + subtype_assigner.Visit(tree) + + +# Map tokens in argument lists to their respective subtype. +_ARGLIST_TOKEN_TO_SUBTYPE = { + '=': subtypes.DEFAULT_OR_NAMED_ASSIGN, + ':': subtypes.TYPED_NAME, + '*': subtypes.VARARGS_STAR, + '**': subtypes.KWARGS_STAR_STAR, +} + + +class _SubtypeAssigner(pytree_visitor.PyTreeVisitor): + """_SubtypeAssigner - see file-level docstring for detailed description. + + The subtype is added as an annotation to the pytree token. + """ + + def Visit_dictsetmaker(self, node): # pylint: disable=invalid-name + # dictsetmaker ::= (test ':' test (comp_for | + # (',' test ':' test)* [','])) | + # (test (comp_for | (',' test)* [','])) + for child in node.children: + self.Visit(child) + + dict_maker = False + + def markAsDictSetGenerator(node): + _AppendFirstLeafTokenSubtype(node, subtypes.DICT_SET_GENERATOR) + for child in node.children: + if pytree_utils.NodeName(child) == 'comp_for': + markAsDictSetGenerator(child) + + for child in node.children: + if pytree_utils.NodeName(child) == 'comp_for': + markAsDictSetGenerator(child) + elif child.type in (grammar_token.COLON, grammar_token.DOUBLESTAR): + dict_maker = True + + if dict_maker: + last_was_colon = False + unpacking = False + for child in node.children: + if pytree_utils.NodeName(child) == 'comp_for': + break + if child.type == grammar_token.DOUBLESTAR: + _AppendFirstLeafTokenSubtype(child, subtypes.KWARGS_STAR_STAR) + if last_was_colon: + if style.Get('INDENT_DICTIONARY_VALUE'): + _InsertPseudoParentheses(child) + else: + _AppendFirstLeafTokenSubtype(child, subtypes.DICTIONARY_VALUE) + elif (isinstance(child, pytree.Node) or + (not child.value.startswith('#') and child.value not in '{:,')): + # Mark the first leaf of a key entry as a DICTIONARY_KEY. We + # normally want to split before them if the dictionary cannot exist + # on a single line. + if not unpacking or pytree_utils.FirstLeafNode(child).value == '**': + _AppendFirstLeafTokenSubtype(child, subtypes.DICTIONARY_KEY) + _AppendSubtypeRec(child, subtypes.DICTIONARY_KEY_PART) + last_was_colon = child.type == grammar_token.COLON + if child.type == grammar_token.DOUBLESTAR: + unpacking = True + elif last_was_colon: + unpacking = False + + def Visit_expr_stmt(self, node): # pylint: disable=invalid-name + # expr_stmt ::= testlist_star_expr (augassign (yield_expr|testlist) + # | ('=' (yield_expr|testlist_star_expr))*) + for child in node.children: + self.Visit(child) + if isinstance(child, pytree.Leaf) and child.value == '=': + _AppendTokenSubtype(child, subtypes.ASSIGN_OPERATOR) + + def Visit_or_test(self, node): # pylint: disable=invalid-name + # or_test ::= and_test ('or' and_test)* + for child in node.children: + self.Visit(child) + if isinstance(child, pytree.Leaf) and child.value == 'or': + _AppendTokenSubtype(child, subtypes.BINARY_OPERATOR) + + def Visit_and_test(self, node): # pylint: disable=invalid-name + # and_test ::= not_test ('and' not_test)* + for child in node.children: + self.Visit(child) + if isinstance(child, pytree.Leaf) and child.value == 'and': + _AppendTokenSubtype(child, subtypes.BINARY_OPERATOR) + + def Visit_not_test(self, node): # pylint: disable=invalid-name + # not_test ::= 'not' not_test | comparison + for child in node.children: + self.Visit(child) + if isinstance(child, pytree.Leaf) and child.value == 'not': + _AppendTokenSubtype(child, subtypes.UNARY_OPERATOR) + + def Visit_comparison(self, node): # pylint: disable=invalid-name + # comparison ::= expr (comp_op expr)* + # comp_op ::= '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not in'|'is'|'is not' + for child in node.children: + self.Visit(child) + if (isinstance(child, pytree.Leaf) and + child.value in {'<', '>', '==', '>=', '<=', '<>', '!=', 'in', 'is'}): + _AppendTokenSubtype(child, subtypes.BINARY_OPERATOR) + elif pytree_utils.NodeName(child) == 'comp_op': + for grandchild in child.children: + _AppendTokenSubtype(grandchild, subtypes.BINARY_OPERATOR) + + def Visit_star_expr(self, node): # pylint: disable=invalid-name + # star_expr ::= '*' expr + for child in node.children: + self.Visit(child) + if isinstance(child, pytree.Leaf) and child.value == '*': + _AppendTokenSubtype(child, subtypes.UNARY_OPERATOR) + _AppendTokenSubtype(child, subtypes.VARARGS_STAR) + + def Visit_expr(self, node): # pylint: disable=invalid-name + # expr ::= xor_expr ('|' xor_expr)* + for child in node.children: + self.Visit(child) + if isinstance(child, pytree.Leaf) and child.value == '|': + _AppendTokenSubtype(child, subtypes.BINARY_OPERATOR) + + def Visit_xor_expr(self, node): # pylint: disable=invalid-name + # xor_expr ::= and_expr ('^' and_expr)* + for child in node.children: + self.Visit(child) + if isinstance(child, pytree.Leaf) and child.value == '^': + _AppendTokenSubtype(child, subtypes.BINARY_OPERATOR) + + def Visit_and_expr(self, node): # pylint: disable=invalid-name + # and_expr ::= shift_expr ('&' shift_expr)* + for child in node.children: + self.Visit(child) + if isinstance(child, pytree.Leaf) and child.value == '&': + _AppendTokenSubtype(child, subtypes.BINARY_OPERATOR) + + def Visit_shift_expr(self, node): # pylint: disable=invalid-name + # shift_expr ::= arith_expr (('<<'|'>>') arith_expr)* + for child in node.children: + self.Visit(child) + if isinstance(child, pytree.Leaf) and child.value in {'<<', '>>'}: + _AppendTokenSubtype(child, subtypes.BINARY_OPERATOR) + + def Visit_arith_expr(self, node): # pylint: disable=invalid-name + # arith_expr ::= term (('+'|'-') term)* + for child in node.children: + self.Visit(child) + if _IsAExprOperator(child): + _AppendTokenSubtype(child, subtypes.BINARY_OPERATOR) + + if _IsSimpleExpression(node): + for child in node.children: + if _IsAExprOperator(child): + _AppendTokenSubtype(child, subtypes.SIMPLE_EXPRESSION) + + def Visit_term(self, node): # pylint: disable=invalid-name + # term ::= factor (('*'|'/'|'%'|'//'|'@') factor)* + for child in node.children: + self.Visit(child) + if _IsMExprOperator(child): + _AppendTokenSubtype(child, subtypes.BINARY_OPERATOR) + + if _IsSimpleExpression(node): + for child in node.children: + if _IsMExprOperator(child): + _AppendTokenSubtype(child, subtypes.SIMPLE_EXPRESSION) + + def Visit_factor(self, node): # pylint: disable=invalid-name + # factor ::= ('+'|'-'|'~') factor | power + for child in node.children: + self.Visit(child) + if isinstance(child, pytree.Leaf) and child.value in '+-~': + _AppendTokenSubtype(child, subtypes.UNARY_OPERATOR) + + def Visit_power(self, node): # pylint: disable=invalid-name + # power ::= atom trailer* ['**' factor] + for child in node.children: + self.Visit(child) + if isinstance(child, pytree.Leaf) and child.value == '**': + _AppendTokenSubtype(child, subtypes.BINARY_OPERATOR) + + def Visit_lambdef(self, node): # pylint: disable=invalid-name + # trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME + _AppendSubtypeRec(node, subtypes.LAMBDEF) + self.DefaultNodeVisit(node) + + def Visit_trailer(self, node): # pylint: disable=invalid-name + for child in node.children: + self.Visit(child) + if isinstance(child, pytree.Leaf) and child.value in '[]': + _AppendTokenSubtype(child, subtypes.SUBSCRIPT_BRACKET) + + def Visit_subscript(self, node): # pylint: disable=invalid-name + # subscript ::= test | [test] ':' [test] [sliceop] + for child in node.children: + self.Visit(child) + if isinstance(child, pytree.Leaf) and child.value == ':': + _AppendTokenSubtype(child, subtypes.SUBSCRIPT_COLON) + + def Visit_sliceop(self, node): # pylint: disable=invalid-name + # sliceop ::= ':' [test] + for child in node.children: + self.Visit(child) + if isinstance(child, pytree.Leaf) and child.value == ':': + _AppendTokenSubtype(child, subtypes.SUBSCRIPT_COLON) + + def Visit_argument(self, node): # pylint: disable=invalid-name + # argument ::= + # test [comp_for] | test '=' test + self._ProcessArgLists(node) + + def Visit_arglist(self, node): # pylint: disable=invalid-name + # arglist ::= + # (argument ',')* (argument [','] + # | '*' test (',' argument)* [',' '**' test] + # | '**' test) + self._ProcessArgLists(node) + _SetArgListSubtype(node, subtypes.DEFAULT_OR_NAMED_ASSIGN, + subtypes.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST) + + def Visit_tname(self, node): # pylint: disable=invalid-name + self._ProcessArgLists(node) + _SetArgListSubtype(node, subtypes.DEFAULT_OR_NAMED_ASSIGN, + subtypes.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST) + + def Visit_decorator(self, node): # pylint: disable=invalid-name + # decorator ::= + # '@' dotted_name [ '(' [arglist] ')' ] NEWLINE + for child in node.children: + if isinstance(child, pytree.Leaf) and child.value == '@': + _AppendTokenSubtype(child, subtype=subtypes.DECORATOR) + self.Visit(child) + + def Visit_funcdef(self, node): # pylint: disable=invalid-name + # funcdef ::= + # 'def' NAME parameters ['->' test] ':' suite + for child in node.children: + if child.type == grammar_token.NAME and child.value != 'def': + _AppendTokenSubtype(child, subtypes.FUNC_DEF) + break + for child in node.children: + self.Visit(child) + + def Visit_parameters(self, node): # pylint: disable=invalid-name + # parameters ::= '(' [typedargslist] ')' + self._ProcessArgLists(node) + if len(node.children) > 2: + _AppendFirstLeafTokenSubtype(node.children[1], subtypes.PARAMETER_START) + _AppendLastLeafTokenSubtype(node.children[-2], subtypes.PARAMETER_STOP) + + def Visit_typedargslist(self, node): # pylint: disable=invalid-name + # typedargslist ::= + # ((tfpdef ['=' test] ',')* + # ('*' [tname] (',' tname ['=' test])* [',' '**' tname] + # | '**' tname) + # | tfpdef ['=' test] (',' tfpdef ['=' test])* [',']) + self._ProcessArgLists(node) + _SetArgListSubtype(node, subtypes.DEFAULT_OR_NAMED_ASSIGN, + subtypes.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST) + tname = False + if not node.children: + return + + _AppendFirstLeafTokenSubtype(node.children[0], subtypes.PARAMETER_START) + _AppendLastLeafTokenSubtype(node.children[-1], subtypes.PARAMETER_STOP) + + tname = pytree_utils.NodeName(node.children[0]) == 'tname' + for i in range(1, len(node.children)): + prev_child = node.children[i - 1] + child = node.children[i] + if prev_child.type == grammar_token.COMMA: + _AppendFirstLeafTokenSubtype(child, subtypes.PARAMETER_START) + elif child.type == grammar_token.COMMA: + _AppendLastLeafTokenSubtype(prev_child, subtypes.PARAMETER_STOP) + + if pytree_utils.NodeName(child) == 'tname': + tname = True + _SetArgListSubtype(child, subtypes.TYPED_NAME, + subtypes.TYPED_NAME_ARG_LIST) + elif child.type == grammar_token.COMMA: + tname = False + elif child.type == grammar_token.EQUAL and tname: + _AppendTokenSubtype(child, subtype=subtypes.TYPED_NAME) + tname = False + + def Visit_varargslist(self, node): # pylint: disable=invalid-name + # varargslist ::= + # ((vfpdef ['=' test] ',')* + # ('*' [vname] (',' vname ['=' test])* [',' '**' vname] + # | '**' vname) + # | vfpdef ['=' test] (',' vfpdef ['=' test])* [',']) + self._ProcessArgLists(node) + for child in node.children: + self.Visit(child) + if isinstance(child, pytree.Leaf) and child.value == '=': + _AppendTokenSubtype(child, subtypes.VARARGS_LIST) + + def Visit_comp_for(self, node): # pylint: disable=invalid-name + # comp_for ::= 'for' exprlist 'in' testlist_safe [comp_iter] + _AppendSubtypeRec(node, subtypes.COMP_FOR) + # Mark the previous node as COMP_EXPR unless this is a nested comprehension + # as these will have the outer comprehension as their previous node. + attr = pytree_utils.GetNodeAnnotation(node.parent, + pytree_utils.Annotation.SUBTYPE) + if not attr or subtypes.COMP_FOR not in attr: + sibling = node.prev_sibling + while sibling: + _AppendSubtypeRec(sibling, subtypes.COMP_EXPR) + sibling = sibling.prev_sibling + self.DefaultNodeVisit(node) + + def Visit_old_comp_for(self, node): # pylint: disable=invalid-name + # Python 3.7 + self.Visit_comp_for(node) + + def Visit_comp_if(self, node): # pylint: disable=invalid-name + # comp_if ::= 'if' old_test [comp_iter] + _AppendSubtypeRec(node, subtypes.COMP_IF) + self.DefaultNodeVisit(node) + + def Visit_old_comp_if(self, node): # pylint: disable=invalid-name + # Python 3.7 + self.Visit_comp_if(node) + + def _ProcessArgLists(self, node): + """Common method for processing argument lists.""" + for child in node.children: + self.Visit(child) + if isinstance(child, pytree.Leaf): + _AppendTokenSubtype( + child, + subtype=_ARGLIST_TOKEN_TO_SUBTYPE.get(child.value, subtypes.NONE)) + + +def _SetArgListSubtype(node, node_subtype, list_subtype): + """Set named assign subtype on elements in a arg list.""" + + def HasSubtype(node): + """Return True if the arg list has a named assign subtype.""" + if isinstance(node, pytree.Leaf): + return node_subtype in pytree_utils.GetNodeAnnotation( + node, pytree_utils.Annotation.SUBTYPE, set()) + + for child in node.children: + node_name = pytree_utils.NodeName(child) + if node_name not in {'atom', 'arglist', 'power'}: + if HasSubtype(child): + return True + + return False + + if not HasSubtype(node): + return + + for child in node.children: + node_name = pytree_utils.NodeName(child) + if node_name not in {'atom', 'COMMA'}: + _AppendFirstLeafTokenSubtype(child, list_subtype) + + +def _AppendTokenSubtype(node, subtype): + """Append the token's subtype only if it's not already set.""" + pytree_utils.AppendNodeAnnotation(node, pytree_utils.Annotation.SUBTYPE, + subtype) + + +def _AppendFirstLeafTokenSubtype(node, subtype): + """Append the first leaf token's subtypes.""" + if isinstance(node, pytree.Leaf): + _AppendTokenSubtype(node, subtype) + return + _AppendFirstLeafTokenSubtype(node.children[0], subtype) + + +def _AppendLastLeafTokenSubtype(node, subtype): + """Append the last leaf token's subtypes.""" + if isinstance(node, pytree.Leaf): + _AppendTokenSubtype(node, subtype) + return + _AppendLastLeafTokenSubtype(node.children[-1], subtype) + + +def _AppendSubtypeRec(node, subtype, force=True): + """Append the leafs in the node to the given subtype.""" + if isinstance(node, pytree.Leaf): + _AppendTokenSubtype(node, subtype) + return + for child in node.children: + _AppendSubtypeRec(child, subtype, force=force) + + +def _InsertPseudoParentheses(node): + """Insert pseudo parentheses so that dicts can be formatted correctly.""" + comment_node = None + if isinstance(node, pytree.Node): + if node.children[-1].type == grammar_token.COMMENT: + comment_node = node.children[-1].clone() + node.children[-1].remove() + + first = pytree_utils.FirstLeafNode(node) + last = pytree_utils.LastLeafNode(node) + + if first == last and first.type == grammar_token.COMMENT: + # A comment was inserted before the value, which is a pytree.Leaf. + # Encompass the dictionary's value into an ATOM node. + last = first.next_sibling + last_clone = last.clone() + new_node = pytree.Node(syms.atom, [first.clone(), last_clone]) + for orig_leaf, clone_leaf in zip(last.leaves(), last_clone.leaves()): + pytree_utils.CopyYapfAnnotations(orig_leaf, clone_leaf) + if hasattr(orig_leaf, 'is_pseudo'): + clone_leaf.is_pseudo = orig_leaf.is_pseudo + + node.replace(new_node) + node = new_node + last.remove() + + first = pytree_utils.FirstLeafNode(node) + last = pytree_utils.LastLeafNode(node) + + lparen = pytree.Leaf( + grammar_token.LPAR, + '(', + context=('', (first.get_lineno(), first.column - 1))) + last_lineno = last.get_lineno() + if last.type == grammar_token.STRING and '\n' in last.value: + last_lineno += last.value.count('\n') + + if last.type == grammar_token.STRING and '\n' in last.value: + last_column = len(last.value.split('\n')[-1]) + 1 + else: + last_column = last.column + len(last.value) + 1 + rparen = pytree.Leaf( + grammar_token.RPAR, ')', context=('', (last_lineno, last_column))) + + lparen.is_pseudo = True + rparen.is_pseudo = True + + if isinstance(node, pytree.Node): + node.insert_child(0, lparen) + node.append_child(rparen) + if comment_node: + node.append_child(comment_node) + _AppendFirstLeafTokenSubtype(node, subtypes.DICTIONARY_VALUE) + else: + clone = node.clone() + for orig_leaf, clone_leaf in zip(node.leaves(), clone.leaves()): + pytree_utils.CopyYapfAnnotations(orig_leaf, clone_leaf) + new_node = pytree.Node(syms.atom, [lparen, clone, rparen]) + node.replace(new_node) + _AppendFirstLeafTokenSubtype(clone, subtypes.DICTIONARY_VALUE) + + +def _IsAExprOperator(node): + return isinstance(node, pytree.Leaf) and node.value in {'+', '-'} + + +def _IsMExprOperator(node): + return isinstance(node, + pytree.Leaf) and node.value in {'*', '/', '%', '//', '@'} + + +def _IsSimpleExpression(node): + """A node with only leafs as children.""" + return all(isinstance(child, pytree.Leaf) for child in node.children) diff --git a/yapf/yapflib/__init__.py b/yapf/yapflib/__init__.py index 67076fe58..e7522b2ca 100644 --- a/yapf/yapflib/__init__.py +++ b/yapf/yapflib/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2015-2016 Google Inc. All Rights Reserved. +# Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/yapf/yapflib/errors.py b/yapf/yapflib/errors.py index e5ab9c2d7..3a0102368 100644 --- a/yapf/yapflib/errors.py +++ b/yapf/yapflib/errors.py @@ -1,4 +1,4 @@ -# Copyright 2015-2016 Google Inc. All Rights Reserved. +# Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,6 +11,30 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +"""YAPF error objects.""" + +from yapf_third_party._ylib2to3.pgen2 import tokenize + + +def FormatErrorMsg(e): + """Convert an exception into a standard format. + + The standard error message format is: + + ::: + + Arguments: + e: An exception. + + Returns: + A properly formatted error message string. + """ + if isinstance(e, SyntaxError): + return '{}:{}:{}: {}'.format(e.filename, e.lineno, e.offset, e.msg) + if isinstance(e, tokenize.TokenError): + return '{}:{}:{}: {}'.format(e.filename, e.args[1][0], e.args[1][1], + e.args[0]) + return '{}:{}:{}: {}'.format(e.args[1][0], e.args[1][1], e.args[1][2], e.msg) class YapfError(Exception): diff --git a/yapf/yapflib/file_resources.py b/yapf/yapflib/file_resources.py index eee93ea20..87b6d863b 100644 --- a/yapf/yapflib/file_resources.py +++ b/yapf/yapflib/file_resources.py @@ -1,4 +1,4 @@ -# Copyright 2015-2016 Google Inc. All Rights Reserved. +# Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,27 +17,95 @@ querying. """ +import codecs import fnmatch import os import re - -from lib2to3.pgen2 import tokenize +import sys +from configparser import ConfigParser +from tokenize import detect_encoding from yapf.yapflib import errors -from yapf.yapflib import py3compat from yapf.yapflib import style +if sys.version_info >= (3, 11): + import tomllib +else: + import tomli as tomllib + +CR = '\r' +LF = '\n' +CRLF = '\r\n' + + +def _GetExcludePatternsFromYapfIgnore(filename): + """Get a list of file patterns to ignore from .yapfignore.""" + ignore_patterns = [] + if os.path.isfile(filename) and os.access(filename, os.R_OK): + with open(filename, 'r') as fd: + for line in fd: + if line.strip() and not line.startswith('#'): + ignore_patterns.append(line.strip()) + + if any(e.startswith('./') for e in ignore_patterns): + raise errors.YapfError('path in .yapfignore should not start with ./') + + return ignore_patterns + + +def _GetExcludePatternsFromPyprojectToml(filename): + """Get a list of file patterns to ignore from pyproject.toml.""" + ignore_patterns = [] + + if os.path.isfile(filename) and os.access(filename, os.R_OK): + with open(filename, 'rb') as fd: + pyproject_toml = tomllib.load(fd) + ignore_patterns = pyproject_toml.get('tool', + {}).get('yapfignore', + {}).get('ignore_patterns', []) + if any(e.startswith('./') for e in ignore_patterns): + raise errors.YapfError('path in pyproject.toml should not start with ./') + + return ignore_patterns + + +def GetExcludePatternsForDir(dirname): + """Return patterns of files to exclude from ignorefile in a given directory. + + Looks for .yapfignore in the directory dirname. + + Arguments: + dirname: (unicode) The name of the directory. + + Returns: + A List of file patterns to exclude if ignore file is found, otherwise empty + List. + """ + ignore_patterns = [] + + yapfignore_file = os.path.join(dirname, '.yapfignore') + if os.path.exists(yapfignore_file): + ignore_patterns += _GetExcludePatternsFromYapfIgnore(yapfignore_file) + + pyproject_toml_file = os.path.join(dirname, 'pyproject.toml') + if os.path.exists(pyproject_toml_file): + ignore_patterns += _GetExcludePatternsFromPyprojectToml(pyproject_toml_file) + return ignore_patterns + -def GetDefaultStyleForDir(dirname): +def GetDefaultStyleForDir(dirname, default_style=style.DEFAULT_STYLE): """Return default style name for a given directory. - Looks for .style.yapf or setup.cfg in the parent directories. + Looks for .style.yapf or setup.cfg or pyproject.toml in the parent + directories. Arguments: dirname: (unicode) The name of the directory. + default_style: The style to return if nothing is found. Defaults to the + global default style ('pep8') unless otherwise specified. Returns: - The filename if found, otherwise return the glboal default (pep8). + The filename if found, otherwise return the default style. """ dirname = os.path.abspath(dirname) while True: @@ -48,23 +116,40 @@ def GetDefaultStyleForDir(dirname): # See if we have a setup.cfg file with a '[yapf]' section. config_file = os.path.join(dirname, style.SETUP_CONFIG) - if os.path.exists(config_file): - with open(config_file) as fd: - config = py3compat.ConfigParser() + try: + fd = open(config_file) + except IOError: + pass # It's okay if it's not there. + else: + with fd: + config = ConfigParser() config.read_file(fd) if config.has_section('yapf'): return config_file - dirname = os.path.dirname(dirname) + # See if we have a pyproject.toml file with a '[tool.yapf]' section. + config_file = os.path.join(dirname, style.PYPROJECT_TOML) + try: + fd = open(config_file, 'rb') + except IOError: + pass # It's okay if it's not there. + else: + with fd: + pyproject_toml = tomllib.load(fd) + style_dict = pyproject_toml.get('tool', {}).get('yapf', None) + if style_dict is not None: + return config_file + if (not dirname or not os.path.basename(dirname) or dirname == os.path.abspath(os.path.sep)): break + dirname = os.path.dirname(dirname) global_file = os.path.expanduser(style.GLOBAL_STYLE) if os.path.exists(global_file): return global_file - return style.DEFAULT_STYLE + return default_style def GetCommandLineFiles(command_line_file_list, recursive, exclude): @@ -72,7 +157,10 @@ def GetCommandLineFiles(command_line_file_list, recursive, exclude): return _FindPythonFiles(command_line_file_list, recursive, exclude) -def WriteReformattedCode(filename, reformatted_code, in_place, encoding): +def WriteReformattedCode(filename, + reformatted_code, + encoding='', + in_place=False): """Emit the reformatted code. Write the reformatted code into the file, if in_place is True. Otherwise, @@ -81,52 +169,98 @@ def WriteReformattedCode(filename, reformatted_code, in_place, encoding): Arguments: filename: (unicode) The name of the unformatted file. reformatted_code: (unicode) The reformatted code. - in_place: (bool) If True, then write the reformatted code to the file. encoding: (unicode) The encoding of the file. + in_place: (bool) If True, then write the reformatted code to the file. """ if in_place: - with py3compat.open_with_encoding( - filename, mode='w', encoding=encoding) as fd: + with codecs.open(filename, mode='w', encoding=encoding) as fd: fd.write(reformatted_code) else: - py3compat.EncodeAndWriteToStdout(reformatted_code, encoding) + sys.stdout.buffer.write(reformatted_code.encode('utf-8')) + + +def LineEnding(lines): + """Retrieve the line ending of the original source.""" + endings = {CRLF: 0, CR: 0, LF: 0} + for line in lines: + if line.endswith(CRLF): + endings[CRLF] += 1 + elif line.endswith(CR): + endings[CR] += 1 + elif line.endswith(LF): + endings[LF] += 1 + return sorted((LF, CRLF, CR), key=endings.get, reverse=True)[0] def _FindPythonFiles(filenames, recursive, exclude): """Find all Python files.""" + if exclude and any(e.startswith('./') for e in exclude): + raise errors.YapfError("path in '--exclude' should not start with ./") + exclude = exclude and [e.rstrip('/' + os.path.sep) for e in exclude] + python_files = [] for filename in filenames: + if filename != '.' and exclude and IsIgnored(filename, exclude): + continue if os.path.isdir(filename): - if recursive: - # TODO(morbo): Look into a version of os.walk that can handle recursion. - python_files.extend( - os.path.join(dirpath, f) - for dirpath, _, filelist in os.walk(filename) for f in filelist - if IsPythonFile(os.path.join(dirpath, f))) - else: + if not recursive: raise errors.YapfError( "directory specified without '--recursive' flag: %s" % filename) + + # TODO(morbo): Look into a version of os.walk that can handle recursion. + excluded_dirs = [] + for dirpath, dirnames, filelist in os.walk(filename): + if dirpath != '.' and exclude and IsIgnored(dirpath, exclude): + excluded_dirs.append(dirpath) + continue + elif any(dirpath.startswith(e) for e in excluded_dirs): + continue + for f in filelist: + filepath = os.path.join(dirpath, f) + if exclude and IsIgnored(filepath, exclude): + continue + if IsPythonFile(filepath): + python_files.append(filepath) + # To prevent it from scanning the contents excluded folders, os.walk() + # lets you amend its list of child dirs `dirnames`. These edits must be + # made in-place instead of creating a modified copy of `dirnames`. + # list.remove() is slow and list.pop() is a headache. Instead clear + # `dirnames` then repopulate it. + dirnames_ = [dirnames.pop(0) for i in range(len(dirnames))] + for dirname in dirnames_: + dir_ = os.path.join(dirpath, dirname) + if IsIgnored(dir_, exclude): + excluded_dirs.append(dir_) + else: + dirnames.append(dirname) + elif os.path.isfile(filename): python_files.append(filename) - if exclude: - return [f for f in python_files - if not any(fnmatch.fnmatch(f, p) for p in exclude)] - return python_files +def IsIgnored(path, exclude): + """Return True if filename matches any patterns in exclude.""" + if exclude is None: + return False + path = path.lstrip(os.path.sep) + while path.startswith('.' + os.path.sep): + path = path[2:] + return any(fnmatch.fnmatch(path, e.rstrip(os.path.sep)) for e in exclude) + + def IsPythonFile(filename): """Return True if filename is a Python file.""" - if os.path.splitext(filename)[1] == '.py': + if os.path.splitext(filename)[1] in frozenset({'.py', '.pyi'}): return True try: with open(filename, 'rb') as fd: - encoding = tokenize.detect_encoding(fd.readline)[0] + encoding = detect_encoding(fd.readline)[0] # Check for correctness of encoding. - with py3compat.open_with_encoding(filename, encoding=encoding) as fd: + with codecs.open(filename, mode='r', encoding=encoding) as fd: fd.read() except UnicodeDecodeError: encoding = 'latin-1' @@ -137,10 +271,15 @@ def IsPythonFile(filename): return False try: - with py3compat.open_with_encoding( - filename, mode='r', encoding=encoding) as fd: - first_line = fd.readlines()[0] - except (IOError, IndexError): + with codecs.open(filename, mode='r', encoding=encoding) as fd: + first_line = fd.readline(256) + except IOError: return False return re.match(r'^#!.*\bpython[23]?\b', first_line) + + +def FileEncoding(filename): + """Return the file's encoding.""" + with open(filename, 'rb') as fd: + return detect_encoding(fd.readline)[0] diff --git a/yapf/yapflib/format_decision_state.py b/yapf/yapflib/format_decision_state.py index 804280ad8..4a6d07cb8 100644 --- a/yapf/yapflib/format_decision_state.py +++ b/yapf/yapflib/format_decision_state.py @@ -1,4 +1,4 @@ -# Copyright 2015-2016 Google Inc. All Rights Reserved. +# Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ """Implements a format decision state object that manages whitespace decisions. Each token is processed one at a time, at which point its whitespace formatting -decisions are made. A graph of potential whitespace formattings is created, +decisions are made. A graph of potential whitespace formatting is created, where each node in the graph is a format decision state object. The heuristic tries formatting the token with and without a newline before it to determine which one has the least penalty. Therefore, the format decision state object for @@ -26,36 +26,35 @@ FormatDecisionState: main class exported by this module. """ -import copy - -from yapf.yapflib import format_token -from yapf.yapflib import split_penalty +from yapf.pytree import split_penalty +from yapf.pytree.pytree_utils import NodeName +from yapf.yapflib import logical_line +from yapf.yapflib import object_state from yapf.yapflib import style -from yapf.yapflib import unwrapped_line - -_COMPOUND_STMTS = frozenset({'for', 'while', 'if', 'elif', 'with', 'except', - 'def', 'class'}) +from yapf.yapflib import subtypes class FormatDecisionState(object): - """The current state when indenting an unwrapped line. + """The current state when indenting a logical line. The FormatDecisionState object is meant to be copied instead of referenced. Attributes: first_indent: The indent of the first token. column: The number of used columns in the current line. + line: The logical line we're currently processing. next_token: The next token to be formatted. paren_level: The level of nesting inside (), [], and {}. - start_of_line_level: The paren_level at the start of this line. lowest_level_on_line: The lowest paren_level on the current line. - newline: Indicates if a newline is added along the edge to this format - decision state node. - previous: The previous format decision state in the decision tree. stack: A stack (of _ParenState) keeping track of properties applying to parenthesis levels. + comp_stack: A stack (of ComprehensionState) keeping track of properties + applying to comprehensions. + param_list_stack: A stack (of ParameterListState) keeping track of + properties applying to function parameter lists. ignore_stack_for_comparison: Ignore the stack of _ParenState for state comparison. + column_limit: The column limit specified by the style. """ def __init__(self, line, first_indent): @@ -65,25 +64,35 @@ def __init__(self, line, first_indent): 'first_indent'. Arguments: - line: (UnwrappedLine) The unwrapped line we're currently processing. + line: (LogicalLine) The logical line we're currently processing. first_indent: (int) The indent of the first token. """ self.next_token = line.first self.column = first_indent self.line = line self.paren_level = 0 - self.start_of_line_level = 0 self.lowest_level_on_line = 0 self.ignore_stack_for_comparison = False self.stack = [_ParenState(first_indent, first_indent)] + self.comp_stack = [] + self.param_list_stack = [] self.first_indent = first_indent - self.newline = False - self.previous = None - self._MoveStateToNextToken() + self.column_limit = style.Get('COLUMN_LIMIT') def Clone(self): - new = copy.copy(self) - new.stack = [copy.copy(state) for state in self.stack] + """Clones a FormatDecisionState object.""" + new = FormatDecisionState(self.line, self.first_indent) + new.next_token = self.next_token + new.column = self.column + new.line = self.line + new.paren_level = self.paren_level + new.line.depth = self.line.depth + new.lowest_level_on_line = self.lowest_level_on_line + new.ignore_stack_for_comparison = self.ignore_stack_for_comparison + new.first_indent = self.first_indent + new.stack = [state.Clone() for state in self.stack] + new.comp_stack = [state.Clone() for state in self.comp_stack] + new.param_list_stack = [state.Clone() for state in self.param_list_stack] return new def __eq__(self, other): @@ -93,28 +102,65 @@ def __eq__(self, other): return (self.next_token == other.next_token and self.column == other.column and self.paren_level == other.paren_level and - self.start_of_line_level == other.start_of_line_level and + self.line.depth == other.line.depth and self.lowest_level_on_line == other.lowest_level_on_line and (self.ignore_stack_for_comparison or - other.ignore_stack_for_comparison or self.stack == other.stack)) + other.ignore_stack_for_comparison or self.stack == other.stack and + self.comp_stack == other.comp_stack and + self.param_list_stack == other.param_list_stack)) def __ne__(self, other): return not self == other def __hash__(self): return hash((self.next_token, self.column, self.paren_level, - self.start_of_line_level, self.lowest_level_on_line)) + self.line.depth, self.lowest_level_on_line)) def __repr__(self): return ('column::%d, next_token::%s, paren_level::%d, stack::[\n\t%s' % (self.column, repr(self.next_token), self.paren_level, '\n\t'.join(repr(s) for s in self.stack) + ']')) - def CanSplit(self): - """Returns True if the line can be split before the next token.""" + def CanSplit(self, must_split): + """Determine if we can split before the next token. + + Arguments: + must_split: (bool) A newline was required before this token. + + Returns: + True if the line can be split before the next token. + """ current = self.next_token + previous = current.previous_token + + if current.is_pseudo: + return False - if current.is_pseudo_paren: + if (not must_split and subtypes.DICTIONARY_KEY_PART in current.subtypes and + subtypes.DICTIONARY_KEY not in current.subtypes and + not style.Get('ALLOW_MULTILINE_DICTIONARY_KEYS')): + # In some situations, a dictionary may be multiline, but pylint doesn't + # like it. So don't allow it unless forced to. + return False + + if (not must_split and subtypes.DICTIONARY_VALUE in current.subtypes and + not style.Get('ALLOW_SPLIT_BEFORE_DICT_VALUE')): + return False + + if previous and previous.value == '(' and current.value == ')': + # Don't split an empty function call list if we aren't splitting before + # dict values. + token = previous.previous_token + while token: + prev = token.previous_token + if not prev or prev.name not in {'NAME', 'DOT'}: + break + token = token.previous_token + if token and subtypes.DICTIONARY_VALUE in token.subtypes: + if not style.Get('ALLOW_SPLIT_BEFORE_DICT_VALUE'): + return False + + if previous and previous.value == '.' and current.value == '.': return False return current.can_break_before @@ -123,140 +169,397 @@ def MustSplit(self): """Returns True if the line must split before the next token.""" current = self.next_token previous = current.previous_token - column_limit = style.Get('COLUMN_LIMIT') + + if current.is_pseudo: + return False if current.must_break_before: return True - if (previous and (style.Get('DEDENT_CLOSING_BRACKETS') or - style.Get('SPLIT_BEFORE_FIRST_ARGUMENT'))): + if not previous: + return False + + if style.Get('SPLIT_ALL_COMMA_SEPARATED_VALUES') and previous.value == ',': + if (subtypes.COMP_FOR in current.subtypes or + subtypes.LAMBDEF in current.subtypes): + return False + + return True + + if (style.Get('FORCE_MULTILINE_DICT') and + subtypes.DICTIONARY_KEY in current.subtypes and not current.is_comment): + return True + + if (style.Get('SPLIT_ALL_TOP_LEVEL_COMMA_SEPARATED_VALUES') and + previous.value == ','): + + if (subtypes.COMP_FOR in current.subtypes or + subtypes.LAMBDEF in current.subtypes): + return False + + # Avoid breaking in a container that fits in the current line if possible + opening = _GetOpeningBracket(current) + + # Can't find opening bracket, behave the same way as + # SPLIT_ALL_COMMA_SEPARATED_VALUES. + if not opening: + return True + + if current.is_comment: + # Don't require splitting before a comment, since it may be related to + # the current line. + return False + + # Allow the fallthrough code to handle the closing bracket. + if current != opening.matching_bracket: + # If the container doesn't fit in the current line, must split + return not self._ContainerFitsOnStartLine(opening) + + if (self.stack[-1].split_before_closing_bracket and + (current.value in '}]' and style.Get('SPLIT_BEFORE_CLOSING_BRACKET') or + current.value in '}])' and style.Get('INDENT_CLOSING_BRACKETS'))): + # Split before the closing bracket if we can. + if (subtypes.SUBSCRIPT_BRACKET not in current.subtypes or + (previous.value == ',' and + not style.Get('DISABLE_ENDING_COMMA_HEURISTIC'))): + return current.node_split_penalty != split_penalty.UNBREAKABLE + + if (current.value == ')' and previous.value == ',' and + not _IsSingleElementTuple(current.matching_bracket)): + return True + + # Prevent splitting before the first argument in compound statements + # with the exception of function declarations. + if (style.Get('SPLIT_BEFORE_FIRST_ARGUMENT') and + _IsCompoundStatement(self.line.first) and + not _IsFunctionDef(self.line.first)): + return False + + ########################################################################### + # List Splitting + if (style.Get('DEDENT_CLOSING_BRACKETS') or + style.Get('INDENT_CLOSING_BRACKETS') or + style.Get('SPLIT_BEFORE_FIRST_ARGUMENT')): bracket = current if current.ClosesScope() else previous - if format_token.Subtype.SUBSCRIPT_BRACKET not in bracket.subtypes: + if subtypes.SUBSCRIPT_BRACKET not in bracket.subtypes: if bracket.OpensScope(): if style.Get('COALESCE_BRACKETS'): if current.OpensScope(): + # Prefer to keep all opening brackets together. return False if (not _IsLastScopeInLine(bracket) or - unwrapped_line.IsSurroundedByBrackets(bracket)): + logical_line.IsSurroundedByBrackets(bracket)): last_token = bracket.matching_bracket else: last_token = _LastTokenInLine(bracket.matching_bracket) - length = last_token.total_length - bracket.total_length - if length + self.column >= column_limit: + if not self._FitsOnLine(bracket, last_token): + # Split before the first element if the whole list can't fit on a + # single line. self.stack[-1].split_before_closing_bracket = True return True - elif style.Get('DEDENT_CLOSING_BRACKETS') and current.ClosesScope(): + elif (style.Get('DEDENT_CLOSING_BRACKETS') or + style.Get('INDENT_CLOSING_BRACKETS')) and current.ClosesScope(): + # Split before and dedent the closing bracket. return self.stack[-1].split_before_closing_bracket - if self.stack[-1].split_before_closing_bracket and current.value in '}]': - # Split if we need to split before the closing bracket. - return current.node_split_penalty != split_penalty.UNBREAKABLE + if (style.Get('SPLIT_BEFORE_EXPRESSION_AFTER_OPENING_PAREN') and + current.is_name): + # An expression that's surrounded by parens gets split after the opening + # parenthesis. + def SurroundedByParens(token): + """Check if it's an expression surrounded by parentheses.""" + while token: + if token.value == ',': + return False + if token.value == ')': + return not token.next_token + if token.OpensScope(): + token = token.matching_bracket.next_token + else: + token = token.next_token + return False + + if (previous.value == '(' and not previous.is_pseudo and + not logical_line.IsSurroundedByBrackets(previous)): + pptoken = previous.previous_token + if (pptoken and not pptoken.is_name and not pptoken.is_keyword and + SurroundedByParens(current)): + return True - if not previous: - return False + if (current.is_name or current.is_string) and previous.value == ',': + # If the list has function calls in it and the full list itself cannot + # fit on the line, then we want to split. Otherwise, we'll get something + # like this: + # + # X = [ + # Bar(xxx='some string', + # yyy='another long string', + # zzz='a third long string'), Bar( + # xxx='some string', + # yyy='another long string', + # zzz='a third long string') + # ] + # + # or when a string formatting syntax. + func_call_or_string_format = False + tok = current.next_token + if current.is_name: + while tok and (tok.is_name or tok.value == '.'): + tok = tok.next_token + func_call_or_string_format = tok and tok.value == '(' + elif current.is_string: + while tok and tok.is_string: + tok = tok.next_token + func_call_or_string_format = tok and tok.value == '%' + if func_call_or_string_format: + open_bracket = logical_line.IsSurroundedByBrackets(current) + if open_bracket: + if open_bracket.value in '[{': + if not self._FitsOnLine(open_bracket, + open_bracket.matching_bracket): + return True + elif tok.value == '(': + if not self._FitsOnLine(current, tok.matching_bracket): + return True + + if (current.OpensScope() and previous.value == ',' and + subtypes.DICTIONARY_KEY not in current.next_token.subtypes): + # If we have a list of tuples, then we can get a similar look as above. If + # the full list cannot fit on the line, then we want a split. + open_bracket = logical_line.IsSurroundedByBrackets(current) + if (open_bracket and open_bracket.value in '[{' and + subtypes.SUBSCRIPT_BRACKET not in open_bracket.subtypes): + if not self._FitsOnLine(current, current.matching_bracket): + return True - # TODO(morbo): This should be controlled with a knob. - if (format_token.Subtype.DICTIONARY_KEY in current.subtypes and - not current.is_comment): - # Place each dictionary entry on its own line. + ########################################################################### + # Dict/Set Splitting + if (style.Get('EACH_DICT_ENTRY_ON_SEPARATE_LINE') and + subtypes.DICTIONARY_KEY in current.subtypes and not current.is_comment): + # Place each dictionary entry onto its own line. + if previous.value == '{' and previous.previous_token: + opening = _GetOpeningBracket(previous.previous_token) + if (opening and opening.value == '(' and opening.previous_token and + opening.previous_token.is_name): + # This is a dictionary that's an argument to a function. + if (self._FitsOnLine(previous, previous.matching_bracket) and + previous.matching_bracket.next_token and + (not opening.matching_bracket.next_token or + opening.matching_bracket.next_token.value != '.') and + _ScopeHasNoCommas(previous)): + # Don't split before the key if: + # - The dictionary fits on a line, and + # - The function call isn't part of a builder-style call and + # - The dictionary has one entry and no trailing comma + return False return True - # TODO(morbo): This should be controlled with a knob. - if format_token.Subtype.DICT_SET_GENERATOR in current.subtypes: + if (style.Get('SPLIT_BEFORE_DICT_SET_GENERATOR') and + subtypes.DICT_SET_GENERATOR in current.subtypes): + # Split before a dict/set generator. return True - if (style.Get('SPLIT_BEFORE_NAMED_ASSIGNS') and - format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST in - current.subtypes): - if (previous.value not in {'=', '*', '**'} and - current.value not in '=,)'): - # If we're going to split the lines because of named arguments, then we - # want to split after the opening bracket as well. But not when this is - # part of function definition. - if not _IsFunctionDefinition(previous): - # Make sure we don't split after the opening bracket if the - # continuation indent is greater than the opening bracket: - # - # a( - # b=1, - # c=2) - indent_amt = self.stack[-1].indent * style.Get('INDENT_WIDTH') - pptoken = previous.previous_token - opening_column = len(pptoken.value) if pptoken else 0 - indent_amt - 1 - if previous.value == '(': - return opening_column >= style.Get('CONTINUATION_INDENT_WIDTH') - opening = _GetOpeningParen(current) - if opening: - arglist_length = (opening.matching_bracket.total_length - - opening.total_length + self.stack[-1].indent) - return arglist_length > column_limit + if (subtypes.DICTIONARY_VALUE in current.subtypes or + (previous.is_pseudo and previous.value == '(' and + not current.is_comment)): + # Split before the dictionary value if we can't fit every dictionary + # entry on its own line. + if not current.OpensScope(): + opening = _GetOpeningBracket(current) + if not self._EachDictEntryFitsOnOneLine(opening): + return style.Get('ALLOW_SPLIT_BEFORE_DICT_VALUE') + + if previous.value == '{': + # Split if the dict/set cannot fit on one line and ends in a comma. + closing = previous.matching_bracket + if (not self._FitsOnLine(previous, closing) and + closing.previous_token.value == ','): + self.stack[-1].split_before_closing_bracket = True + return True + + ########################################################################### + # Argument List Splitting if style.Get('SPLIT_ARGUMENTS_WHEN_COMMA_TERMINATED'): # Split before arguments in a function call or definition if the # arguments are terminated by a comma. - opening = _GetOpeningParen(current) + opening = _GetOpeningBracket(current) if opening and opening.previous_token and opening.previous_token.is_name: if previous.value in '(,': if opening.matching_bracket.previous_token.value == ',': return True - if (previous.value in '{[' and current.lineno != previous.lineno and - format_token.Subtype.SUBSCRIPT_BRACKET not in previous.subtypes): - # Retain the split after the container opening. - return True - - if (previous.value == ':' and _IsDictionaryValue(current) and - current.lineno != previous.lineno): - # Retain the split between the dictionary key and value. + if (style.Get('SPLIT_BEFORE_NAMED_ASSIGNS') and not current.is_comment and + subtypes.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST in current.subtypes): + if (previous.value not in {'=', ':', '*', '**'} and + current.value not in ':=,)' and not _IsFunctionDefinition(previous)): + # If we're going to split the lines because of named arguments, then we + # want to split after the opening bracket as well. But not when this is + # part of a function definition. + if previous.value == '(': + # Make sure we don't split after the opening bracket if the + # continuation indent is greater than the opening bracket: + # + # a( + # b=1, + # c=2) + if (self._FitsOnLine(previous, previous.matching_bracket) and + logical_line.IsSurroundedByBrackets(previous)): + # An argument to a function is a function call with named + # assigns. + return False + + # Don't split if not required + if (not style.Get('SPLIT_BEFORE_EXPRESSION_AFTER_OPENING_PAREN') and + not style.Get('SPLIT_BEFORE_FIRST_ARGUMENT')): + return False + + column = self.column - self.stack[-1].last_space + return column > style.Get('CONTINUATION_INDENT_WIDTH') + + opening = _GetOpeningBracket(current) + if opening: + return not self._ContainerFitsOnStartLine(opening) + + if (current.value not in '{)' and previous.value == '(' and + self._ArgumentListHasDictionaryEntry(current)): return True - if previous.value == '{': - closing = previous.matching_bracket - length = closing.total_length - previous.total_length + self.column - if length > column_limit and closing.previous_token.value == ',': - self.stack[-1].split_before_closing_bracket = True - return True + if ((current.is_name or current.value in {'*', '**'}) and + previous.value == ','): + # If we have a function call within an argument list and it won't fit on + # the remaining line, but it will fit on a line by itself, then go ahead + # and split before the call. + opening = _GetOpeningBracket(current) + if (opening and opening.value == '(' and opening.previous_token and + (opening.previous_token.is_name or + opening.previous_token.value in {'*', '**'})): + is_func_call = False + opening = current + while opening: + if opening.value == '(': + is_func_call = True + break + if (not (opening.is_name or opening.value in {'*', '**'}) and + opening.value != '.'): + break + opening = opening.next_token + + if is_func_call: + if (not self._FitsOnLine(current, opening.matching_bracket) or + (opening.matching_bracket.next_token and + opening.matching_bracket.next_token.value != ',' and + not opening.matching_bracket.next_token.ClosesScope())): + return True - if (format_token.Subtype.COMP_FOR in current.subtypes and - format_token.Subtype.COMP_FOR not in previous.subtypes): - # Split at the beginning of a list comprehension. - length = _GetLengthOfSubtype(current, format_token.Subtype.COMP_FOR, - format_token.Subtype.COMP_IF) - if length + self.column > column_limit: + pprevious = previous.previous_token + + # A function call with a dictionary as its first argument may result in + # unreadable formatting if the dictionary spans multiple lines. The + # dictionary itself is formatted just fine, but the remaining arguments are + # indented too far: + # + # function_call({ + # KEY_1: 'value one', + # KEY_2: 'value two', + # }, + # default=False) + if (current.value == '{' and previous.value == '(' and pprevious and + pprevious.is_name): + dict_end = current.matching_bracket + next_token = dict_end.next_token + if next_token.value == ',' and not self._FitsOnLine(current, dict_end): return True - if (format_token.Subtype.COMP_IF in current.subtypes and - format_token.Subtype.COMP_IF not in previous.subtypes): - # Split at the beginning of an if expression. - length = _GetLengthOfSubtype(current, format_token.Subtype.COMP_IF) - if length + self.column > column_limit: - return True + if (current.is_name and pprevious and pprevious.is_name and + previous.value == '('): - previous_previous_token = previous.previous_token - if (current.is_name and previous_previous_token and - previous_previous_token.is_name and previous.value == '('): - arg_length = previous.matching_bracket.total_length - arg_length -= previous.total_length - if arg_length + self.column > column_limit: - if _IsFunctionCallWithArguments(current): - # There is a function call, with more than 1 argument, where - # the first argument is itself a function call with arguments. - # In this specific case, if we split after the first argument's - # opening '(', then the formatting will look bad for the rest - # of the arguments. Instead, enforce a split before that - # argument to keep things looking good. + if (not self._FitsOnLine(previous, previous.matching_bracket) and + _IsFunctionCallWithArguments(current)): + # There is a function call, with more than 1 argument, where the first + # argument is itself a function call with arguments that does not fit + # into the line. In this specific case, if we split after the first + # argument's opening '(', then the formatting will look bad for the + # rest of the arguments. E.g.: + # + # outer_function_call(inner_function_call( + # inner_arg1, inner_arg2), + # outer_arg1, outer_arg2) + # + # Instead, enforce a split before that argument to keep things looking + # good. + if (style.Get('SPLIT_BEFORE_EXPRESSION_AFTER_OPENING_PAREN') or + style.Get('SPLIT_BEFORE_FIRST_ARGUMENT')): return True - elif current.OpensScope(): - arg_length = current.matching_bracket.total_length - arg_length -= current.total_length - if arg_length + self.column > column_limit: - # There is a data literal that will need to be split and could mess - # up the formatting. + + opening = _GetOpeningBracket(current) + if (opening and opening.value == '(' and opening.previous_token and + (opening.previous_token.is_name or + opening.previous_token.value in {'*', '**'})): + is_func_call = False + opening = current + while opening: + if opening.value == '(': + is_func_call = True + break + if (not (opening.is_name or opening.value in {'*', '**'}) and + opening.value != '.'): + break + opening = opening.next_token + + if is_func_call: + if (not self._FitsOnLine(current, opening.matching_bracket) or + (opening.matching_bracket.next_token and + opening.matching_bracket.next_token.value != ',' and + not opening.matching_bracket.next_token.ClosesScope())): + return True + + if (previous.OpensScope() and not current.OpensScope() and + not current.is_comment and + subtypes.SUBSCRIPT_BRACKET not in previous.subtypes): + if pprevious and not pprevious.is_keyword and not pprevious.is_name: + # We want to split if there's a comment in the container. + token = current + while token != previous.matching_bracket: + if token.is_comment: + return True + token = token.next_token + if previous.value == '(': + pptoken = previous.previous_token + if not pptoken or not pptoken.is_name: + # Split after the opening of a tuple if it doesn't fit on the current + # line and it's not a function call. + if self._FitsOnLine(previous, previous.matching_bracket): + return False + elif not self._FitsOnLine(previous, previous.matching_bracket): + if len(previous.container_elements) == 1: + return False + + elements = previous.container_elements + [previous.matching_bracket] + i = 1 + while i < len(elements): + if (not elements[i - 1].OpensScope() and + not self._FitsOnLine(elements[i - 1], elements[i])): + return True + i += 1 + + if (self.column_limit - self.column) / float(self.column_limit) < 0.3: + # Try not to squish all of the arguments off to the right. return True + else: + # Split after the opening of a container if it doesn't fit on the + # current line. + if not self._FitsOnLine(previous, previous.matching_bracket): + return True + ########################################################################### + # Original Formatting Splitting + # These checks rely upon the original formatting. This is in order to + # attempt to keep hand-written code in the same condition as it was before. + # However, this may cause the formatter to fail to be idempotent. if (style.Get('SPLIT_BEFORE_BITWISE_OPERATOR') and current.value in '&|' and previous.lineno < current.lineno): # Retain the split before a bitwise operator. @@ -264,7 +567,7 @@ def MustSplit(self): if (current.is_comment and previous.lineno < current.lineno - current.value.count('\n')): - # If a comment comes in the middle of an unwrapped line (like an if + # If a comment comes in the middle of a logical line (like an if # conditional with comments interspersed), then we want to split if the # original comments were on a separate line. return True @@ -286,13 +589,18 @@ def AddTokenToState(self, newline, dry_run, must_split=False): Returns: The penalty of splitting after the current token. """ + self._PushParameterListState(newline) + penalty = 0 if newline: penalty = self._AddTokenOnNewline(dry_run, must_split) else: self._AddTokenOnCurrentLine(dry_run) - return self._MoveStateToNextToken() + penalty + penalty += self._CalculateComprehensionState(newline) + penalty += self._CalculateParameterListState(newline) + + return self.MoveStateToNextToken() + penalty def _AddTokenOnCurrentLine(self, dry_run): """Puts the token on the current line. @@ -307,6 +615,11 @@ def _AddTokenOnCurrentLine(self, dry_run): previous = current.previous_token spaces = current.spaces_required_before + if isinstance(spaces, list): + # Don't set the value here, as we need to look at the lines near + # this one to determine the actual horizontal alignment value. + spaces = 0 + if not dry_run: current.AddWhitespacePrefix(newlines_before=0, spaces=spaces) @@ -347,18 +660,24 @@ def _AddTokenOnNewline(self, dry_run, must_split): self.column = self._GetNewlineColumn() if not dry_run: - current.AddWhitespacePrefix(newlines_before=1, spaces=self.column) + indent_level = self.line.depth + spaces = self.column + if spaces: + spaces -= indent_level * style.Get('INDENT_WIDTH') + current.AddWhitespacePrefix( + newlines_before=1, spaces=spaces, indent_level=indent_level) if not current.is_comment: self.stack[-1].last_space = self.column - self.start_of_line_level = self.paren_level self.lowest_level_on_line = self.paren_level - if (previous.OpensScope() or (previous.is_comment and - previous.previous_token is not None and - previous.previous_token.OpensScope())): - self.stack[-1].closing_scope_indent = max( - 0, self.stack[-1].indent - style.Get('CONTINUATION_INDENT_WIDTH')) + if (previous.OpensScope() or + (previous.is_comment and previous.previous_token is not None and + previous.previous_token.OpensScope())): + dedent = (style.Get('CONTINUATION_INDENT_WIDTH'), + 0)[style.Get('INDENT_CLOSING_BRACKETS')] + self.stack[-1].closing_scope_indent = ( + max(0, self.stack[-1].indent - dedent)) self.stack[-1].split_before_closing_bracket = True # Calculate the split penalty. @@ -368,60 +687,29 @@ def _AddTokenOnNewline(self, dry_run, must_split): # Don't penalize for a must split. return penalty - if previous.is_pseudo_paren and previous.value == '(': + if previous.is_pseudo and previous.value == '(': # Small penalty for splitting after a pseudo paren. penalty += 50 # Add a penalty for each increasing newline we add, but don't penalize for # splitting before an if-expression or list comprehension. - if not must_split and current.value not in {'if', 'for'}: + if current.value not in {'if', 'for'}: last = self.stack[-1] last.num_line_splits += 1 - penalty += (style.Get('SPLIT_PENALTY_FOR_ADDED_LINE_SPLIT') * - last.num_line_splits) + penalty += ( + style.Get('SPLIT_PENALTY_FOR_ADDED_LINE_SPLIT') * + last.num_line_splits) - return penalty + 10 + if current.OpensScope() and previous.OpensScope(): + # Prefer to keep opening brackets coalesced (unless it's at the beginning + # of a function call). + pprev = previous.previous_token + if not pprev or not pprev.is_name: + penalty += 10 - def _GetNewlineColumn(self): - """Return the new column on the newline.""" - current = self.next_token - previous = current.previous_token - top_of_stack = self.stack[-1] - - if current.spaces_required_before > 2 or self.line.disable: - return current.spaces_required_before - - if current.OpensScope(): - return top_of_stack.indent if self.paren_level else self.first_indent - - if current.ClosesScope(): - if (previous.OpensScope() or (previous.is_comment and - previous.previous_token is not None and - previous.previous_token.OpensScope())): - return max(0, - top_of_stack.indent - style.Get('CONTINUATION_INDENT_WIDTH')) - return top_of_stack.closing_scope_indent - - if (previous and previous.is_string and current.is_string and - format_token.Subtype.DICTIONARY_VALUE in current.subtypes): - return previous.column - - if style.Get('INDENT_DICTIONARY_VALUE'): - if previous and (previous.value == ':' or previous.is_pseudo_paren): - if format_token.Subtype.DICTIONARY_VALUE in current.subtypes: - return top_of_stack.indent - - if (self.line.first.value in _COMPOUND_STMTS and - (not style.Get('DEDENT_CLOSING_BRACKETS') or - style.Get('SPLIT_BEFORE_FIRST_ARGUMENT'))): - token_indent = (len(self.line.first.whitespace_prefix.split('\n')[-1]) + - style.Get('INDENT_WIDTH')) - if token_indent == top_of_stack.indent: - return top_of_stack.indent + style.Get('CONTINUATION_INDENT_WIDTH') - - return top_of_stack.indent + return penalty + 10 - def _MoveStateToNextToken(self): + def MoveStateToNextToken(self): """Calculate format decision state information and move onto the next token. Before moving onto the next token, we first calculate the format decision @@ -448,7 +736,10 @@ def _MoveStateToNextToken(self): # If we encounter a closing bracket, we can remove a level from our # parenthesis stack. if len(self.stack) > 1 and current.ClosesScope(): - self.stack[-2].last_space = self.stack[-1].last_space + if subtypes.DICTIONARY_KEY_PART in current.subtypes: + self.stack[-2].last_space = self.stack[-2].indent + else: + self.stack[-2].last_space = self.stack[-1].last_space self.stack.pop() self.paren_level -= 1 @@ -456,16 +747,16 @@ def _MoveStateToNextToken(self): if is_multiline_string: # This is a multiline string. Only look at the first line. self.column += len(current.value.split('\n')[0]) - elif not current.is_pseudo_paren or current.value == '(': + elif not current.is_pseudo: self.column += len(current.value) self.next_token = self.next_token.next_token # Calculate the penalty for overflowing the column limit. penalty = 0 - column_limit = style.Get('COLUMN_LIMIT') - if not current.is_pylint_comment and self.column > column_limit: - excess_characters = self.column - column_limit + if (not current.is_pylint_comment and not current.is_pytype_comment and + not current.is_copybara_comment and self.column > self.column_limit): + excess_characters = self.column - self.column_limit penalty += style.Get('SPLIT_PENALTY_EXCESS_CHARACTER') * excess_characters if is_multiline_string: @@ -475,45 +766,414 @@ def _MoveStateToNextToken(self): return penalty + def _CalculateComprehensionState(self, newline): + """Makes required changes to comprehension state. + + Args: + newline: Whether the current token is to be added on a newline. + + Returns: + The penalty for the token-newline combination given the current + comprehension state. + """ + current = self.next_token + previous = current.previous_token + top_of_stack = self.comp_stack[-1] if self.comp_stack else None + penalty = 0 + + if top_of_stack is not None: + # Check if the token terminates the current comprehension. + if current == top_of_stack.closing_bracket: + last = self.comp_stack.pop() + # Lightly penalize comprehensions that are split across multiple lines. + if last.has_interior_split: + penalty += style.Get('SPLIT_PENALTY_COMPREHENSION') + + return penalty + + if newline: + top_of_stack.has_interior_split = True + + if (subtypes.COMP_EXPR in current.subtypes and + subtypes.COMP_EXPR not in previous.subtypes): + self.comp_stack.append(object_state.ComprehensionState(current)) + return penalty + + if current.value == 'for' and subtypes.COMP_FOR in current.subtypes: + if top_of_stack.for_token is not None: + # Treat nested comprehensions like normal comp_if expressions. + # Example: + # my_comp = [ + # a.qux + b.qux + # for a in foo + # --> for b in bar <-- + # if a.zut + b.zut + # ] + if (style.Get('SPLIT_COMPLEX_COMPREHENSION') and + top_of_stack.has_split_at_for != newline and + (top_of_stack.has_split_at_for or + not top_of_stack.HasTrivialExpr())): + penalty += split_penalty.UNBREAKABLE + else: + top_of_stack.for_token = current + top_of_stack.has_split_at_for = newline + + # Try to keep trivial expressions on the same line as the comp_for. + if (style.Get('SPLIT_COMPLEX_COMPREHENSION') and newline and + top_of_stack.HasTrivialExpr()): + penalty += split_penalty.CONNECTED + + if (subtypes.COMP_IF in current.subtypes and + subtypes.COMP_IF not in previous.subtypes): + # Penalize breaking at comp_if when it doesn't match the newline structure + # in the rest of the comprehension. + if (style.Get('SPLIT_COMPLEX_COMPREHENSION') and + top_of_stack.has_split_at_for != newline and + (top_of_stack.has_split_at_for or not top_of_stack.HasTrivialExpr())): + penalty += split_penalty.UNBREAKABLE + + return penalty + + def _PushParameterListState(self, newline): + """Push a new parameter list state for a function definition. + + Args: + newline: Whether the current token is to be added on a newline. + """ + current = self.next_token + previous = current.previous_token + + if _IsFunctionDefinition(previous): + first_param_column = previous.total_length + self.stack[-2].indent + self.param_list_stack.append( + object_state.ParameterListState(previous, newline, + first_param_column)) + + def _CalculateParameterListState(self, newline): + """Makes required changes to parameter list state. + + Args: + newline: Whether the current token is to be added on a newline. + + Returns: + The penalty for the token-newline combination given the current + parameter state. + """ + current = self.next_token + previous = current.previous_token + penalty = 0 + + if _IsFunctionDefinition(previous): + first_param_column = previous.total_length + self.stack[-2].indent + if not newline: + param_list = self.param_list_stack[-1] + if param_list.parameters and param_list.has_typed_return: + last_param = param_list.parameters[-1].first_token + last_token = _LastTokenInLine(previous.matching_bracket) + total_length = last_token.total_length + total_length -= last_param.total_length - len(last_param.value) + if total_length + self.column > self.column_limit: + # If we need to split before the trailing code of a function + # definition with return types, then also split before the opening + # parameter so that the trailing bit isn't indented on a line by + # itself: + # + # def rrrrrrrrrrrrrrrrrrrrrr(ccccccccccccccccccccccc: Tuple[Text] + # ) -> List[Tuple[Text, Text]]: + # pass + penalty += split_penalty.VERY_STRONGLY_CONNECTED + return penalty + + if first_param_column <= self.column: + # Make sure we don't split after the opening bracket if the + # continuation indent is greater than the opening bracket: + # + # a( + # b=1, + # c=2) + penalty += split_penalty.VERY_STRONGLY_CONNECTED + return penalty + + if not self.param_list_stack: + return penalty + + param_list = self.param_list_stack[-1] + if current == self.param_list_stack[-1].closing_bracket: + self.param_list_stack.pop() # We're done with this state. + if newline and param_list.has_typed_return: + if param_list.split_before_closing_bracket: + penalty -= split_penalty.STRONGLY_CONNECTED + elif param_list.LastParamFitsOnLine(self.column): + penalty += split_penalty.STRONGLY_CONNECTED + + if (not newline and param_list.has_typed_return and + param_list.has_split_before_first_param): + # Prefer splitting before the closing bracket if there's a return type + # and we've already split before the first parameter. + penalty += split_penalty.STRONGLY_CONNECTED + + return penalty + + if not param_list.parameters: + return penalty + + if newline: + if self._FitsOnLine(param_list.parameters[0].first_token, + _LastTokenInLine(param_list.closing_bracket)): + penalty += split_penalty.STRONGLY_CONNECTED + + if (not newline and style.Get('SPLIT_BEFORE_NAMED_ASSIGNS') and + param_list.has_default_values and + current != param_list.parameters[0].first_token and + current != param_list.closing_bracket and + subtypes.PARAMETER_START in current.subtypes): + # If we want to split before parameters when there are named assigns, + # then add a penalty for not splitting. + penalty += split_penalty.STRONGLY_CONNECTED + + return penalty + + def _IndentWithContinuationAlignStyle(self, column): + if column == 0: + return column + align_style = style.Get('CONTINUATION_ALIGN_STYLE') + if align_style == 'FIXED': + return ((self.line.depth * style.Get('INDENT_WIDTH')) + + style.Get('CONTINUATION_INDENT_WIDTH')) + if align_style == 'VALIGN-RIGHT': + indent_width = style.Get('INDENT_WIDTH') + return indent_width * int((column + indent_width - 1) / indent_width) + return column + + def _GetNewlineColumn(self): + """Return the new column on the newline.""" + current = self.next_token + previous = current.previous_token + top_of_stack = self.stack[-1] + + if isinstance(current.spaces_required_before, list): + # Don't set the value here, as we need to look at the lines near + # this one to determine the actual horizontal alignment value. + return 0 + elif current.spaces_required_before > 2 or self.line.disable: + return current.spaces_required_before + + cont_aligned_indent = self._IndentWithContinuationAlignStyle( + top_of_stack.indent) + + if current.OpensScope(): + return cont_aligned_indent if self.paren_level else self.first_indent + + if current.ClosesScope(): + if (previous.OpensScope() or + (previous.is_comment and previous.previous_token is not None and + previous.previous_token.OpensScope())): + return max(0, + top_of_stack.indent - style.Get('CONTINUATION_INDENT_WIDTH')) + return top_of_stack.closing_scope_indent + + if (previous and previous.is_string and current.is_string and + subtypes.DICTIONARY_VALUE in current.subtypes): + return previous.column + + if style.Get('INDENT_DICTIONARY_VALUE'): + if previous and (previous.value == ':' or previous.is_pseudo): + if subtypes.DICTIONARY_VALUE in current.subtypes: + return top_of_stack.indent + + if (not self.param_list_stack and _IsCompoundStatement(self.line.first) and + (not (style.Get('DEDENT_CLOSING_BRACKETS') or + style.Get('INDENT_CLOSING_BRACKETS')) or + style.Get('SPLIT_BEFORE_FIRST_ARGUMENT'))): + token_indent = ( + len(self.line.first.whitespace_prefix.split('\n')[-1]) + + style.Get('INDENT_WIDTH')) + if token_indent == top_of_stack.indent: + return token_indent + style.Get('CONTINUATION_INDENT_WIDTH') + + if (self.param_list_stack and + not self.param_list_stack[-1].SplitBeforeClosingBracket( + top_of_stack.indent) and top_of_stack.indent + == ((self.line.depth + 1) * style.Get('INDENT_WIDTH'))): + # NOTE: comment inside argument list is not excluded in subtype assigner + if (subtypes.PARAMETER_START in current.subtypes or + (previous.is_comment and + subtypes.PARAMETER_START in previous.subtypes)): + return top_of_stack.indent + style.Get('CONTINUATION_INDENT_WIDTH') + + return cont_aligned_indent + + def _FitsOnLine(self, start, end): + """Determines if line between start and end can fit on the current line.""" + length = end.total_length - start.total_length + if not start.is_pseudo: + length += len(start.value) + return length + self.column <= self.column_limit + + def _EachDictEntryFitsOnOneLine(self, opening): + """Determine if each dict elems can fit on one line.""" + + def PreviousNonCommentToken(tok): + tok = tok.previous_token + while tok.is_comment: + tok = tok.previous_token + return tok + + def ImplicitStringConcatenation(tok): + num_strings = 0 + if tok.is_pseudo: + tok = tok.next_token + while tok.is_string: + num_strings += 1 + tok = tok.next_token + return num_strings > 1 + + def DictValueIsContainer(opening, closing): + """Return true if the dictionary value is a container.""" + if not opening or not closing: + return False + colon = opening.previous_token + while colon: + if not colon.is_pseudo: + break + colon = colon.previous_token + if not colon or colon.value != ':': + return False + key = colon.previous_token + if not key: + return False + return subtypes.DICTIONARY_KEY_PART in key.subtypes + + closing = opening.matching_bracket + entry_start = opening.next_token + current = opening.next_token.next_token + + while current and current != closing: + if subtypes.DICT_SET_GENERATOR in current.subtypes: + break + if subtypes.DICTIONARY_KEY in current.subtypes: + prev = PreviousNonCommentToken(current) + if prev.value == ',': + prev = PreviousNonCommentToken(prev.previous_token) + if not DictValueIsContainer(prev.matching_bracket, prev): + length = prev.total_length - entry_start.total_length + length += len(entry_start.value) + if length + self.stack[-2].indent >= self.column_limit: + return False + entry_start = current + if current.OpensScope(): + if ((current.value == '{' or + (current.is_pseudo and current.next_token.value == '{') and + subtypes.DICTIONARY_VALUE in current.subtypes) or + ImplicitStringConcatenation(current)): + # A dictionary entry that cannot fit on a single line shouldn't matter + # to this calculation. If it can't fit on a single line, then the + # opening should be on the same line as the key and the rest on + # newlines after it. But the other entries should be on single lines + # if possible. + if current.matching_bracket: + current = current.matching_bracket + while current: + if current == closing: + return True + if subtypes.DICTIONARY_KEY in current.subtypes: + entry_start = current + break + current = current.next_token + else: + current = current.matching_bracket + else: + current = current.next_token + + # At this point, current is the closing bracket. Go back one to get the end + # of the dictionary entry. + current = PreviousNonCommentToken(current) + length = current.total_length - entry_start.total_length + length += len(entry_start.value) + return length + self.stack[-2].indent <= self.column_limit + + def _ArgumentListHasDictionaryEntry(self, token): + """Check if the function argument list has a dictionary as an arg.""" + if _IsArgumentToFunction(token): + while token: + if token.value == '{': + length = token.matching_bracket.total_length - token.total_length + return length + self.stack[-2].indent > self.column_limit + if token.ClosesScope(): + break + if token.OpensScope(): + token = token.matching_bracket + token = token.next_token + return False + + def _ContainerFitsOnStartLine(self, opening): + """Check if the container can fit on its starting line.""" + return (opening.matching_bracket.total_length - opening.total_length + + self.stack[-1].indent) <= self.column_limit + + +_COMPOUND_STMTS = frozenset({ + 'for', + 'while', + 'if', + 'elif', + 'with', + 'except', + 'def', + 'class', +}) + + +def _IsCompoundStatement(token): + value = token.value + if value == 'async': + token = token.next_token + if token.value in _COMPOUND_STMTS: + return True + parent_name = NodeName(token.node.parent) + return value == 'match' and parent_name == 'match_stmt' or \ + value == 'case' and parent_name == 'case_stmt' + + +def _IsFunctionDef(token): + if token.value == 'async': + token = token.next_token + return token.value == 'def' + def _IsFunctionCallWithArguments(token): while token: if token.value == '(': token = token.next_token return token and token.value != ')' - elif token.name not in {'NAME', 'DOT'}: + elif token.name not in {'NAME', 'DOT', 'EQUAL'}: break token = token.next_token return False -def _IsDictionaryValue(token): - while token: - if format_token.Subtype.DICTIONARY_VALUE in token.subtypes: - return True - token = token.previous_token - return False - +def _IsArgumentToFunction(token): + bracket = logical_line.IsSurroundedByBrackets(token) + if not bracket or bracket.value != '(': + return False + previous = bracket.previous_token + return previous and previous.is_name -def _GetLengthOfSubtype(token, subtype, exclude=None): - current = token - while (current.next_token and subtype in current.subtypes and - (exclude is None or exclude not in current.subtypes)): - current = current.next_token - return current.total_length - token.total_length + 1 +def _GetOpeningBracket(current): + """Get the opening bracket containing the current token.""" + if current.matching_bracket and not current.is_pseudo: + return current if current.OpensScope() else current.matching_bracket -def _GetOpeningParen(current): - previous = current - if previous and previous.matching_bracket: - return previous.matching_bracket - while previous is not None and previous.matching_bracket is None: - previous = previous.previous_token - if not previous: - break - if previous.ClosesScope(): - previous = previous.matching_bracket.previous_token - return previous + while current: + if current.ClosesScope(): + current = current.matching_bracket + elif current.is_pseudo: + current = current.previous_token + elif current.OpensScope(): + return current + current = current.previous_token + return None def _LastTokenInLine(current): @@ -524,11 +1184,11 @@ def _LastTokenInLine(current): def _IsFunctionDefinition(current): prev = current.previous_token - return (current.value == '(' and prev and - format_token.Subtype.FUNC_DEF in prev.subtypes) + return current.value == '(' and prev and subtypes.FUNC_DEF in prev.subtypes def _IsLastScopeInLine(current): + current = current.matching_bracket while current: current = current.next_token if current and current.OpensScope(): @@ -536,6 +1196,29 @@ def _IsLastScopeInLine(current): return True +def _IsSingleElementTuple(token): + """Check if it's a single-element tuple.""" + close = token.matching_bracket + token = token.next_token + num_commas = 0 + while token != close: + if token.value == ',': + num_commas += 1 + token = token.matching_bracket if token.OpensScope() else token.next_token + return num_commas == 1 + + +def _ScopeHasNoCommas(token): + """Check if the scope has no commas.""" + close = token.matching_bracket + token = token.next_token + while token != close: + if token.value == ',': + return False + token = token.matching_bracket if token.OpensScope() else token.next_token + return True + + class _ParenState(object): """Maintains the state of the bracket enclosures. @@ -546,6 +1229,7 @@ class _ParenState(object): indent: The column position to which a specified parenthesis level needs to be indented. last_space: The column position of the last space on each level. + closing_scope_indent: The column position of the closing indentation. split_before_closing_bracket: Whether a newline needs to be inserted before the closing bracket. We only want to insert a newline before the closing bracket if there also was a newline after the beginning left bracket. @@ -562,7 +1246,7 @@ def __init__(self, indent, last_space): self.split_before_closing_bracket = False self.num_line_splits = 0 - def __copy__(self): + def Clone(self): state = _ParenState(self.indent, self.last_space) state.closing_scope_indent = self.closing_scope_indent state.split_before_closing_bracket = self.split_before_closing_bracket @@ -572,3 +1256,13 @@ def __copy__(self): def __repr__(self): return '[indent::%d, last_space::%d, closing_scope_indent::%d]' % ( self.indent, self.last_space, self.closing_scope_indent) + + def __eq__(self, other): + return hash(self) == hash(other) + + def __ne__(self, other): + return not self == other + + def __hash__(self, *args, **kwargs): + return hash((self.indent, self.last_space, self.closing_scope_indent, + self.split_before_closing_bracket, self.num_line_splits)) diff --git a/yapf/yapflib/format_token.py b/yapf/yapflib/format_token.py index 32d11c9da..5abcb8c3f 100644 --- a/yapf/yapflib/format_token.py +++ b/yapf/yapflib/format_token.py @@ -1,4 +1,4 @@ -# Copyright 2015-2016 Google Inc. All Rights Reserved. +# Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,61 +11,63 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Pytree nodes with extra formatting information. - -This is a thin wrapper around a pytree.Leaf node. -""" +"""Enhanced token information for formatting.""" import keyword import re +from functools import lru_cache -from lib2to3 import pytree -from lib2to3.pgen2 import token +from yapf_third_party._ylib2to3.pgen2 import token +from yapf_third_party._ylib2to3.pytree import type_repr -from yapf.yapflib import py3compat -from yapf.yapflib import pytree_utils +from yapf.pytree import pytree_utils from yapf.yapflib import style +from yapf.yapflib import subtypes CONTINUATION = token.N_TOKENS -token.N_TOKENS += 1 +_OPENING_BRACKETS = frozenset({'(', '[', '{'}) +_CLOSING_BRACKETS = frozenset({')', ']', '}'}) + + +def _TabbedContinuationAlignPadding(spaces, align_style, tab_width): + """Build padding string for continuation alignment in tabbed indentation. -class Subtype(object): - """Subtype information about tokens. + Arguments: + spaces: (int) The number of spaces to place before the token for alignment. + align_style: (str) The alignment style for continuation lines. + tab_width: (int) Number of columns of each tab character. - Gleaned from parsing the code. Helps determine the best formatting. + Returns: + A padding string for alignment with style specified by align_style option. """ - NONE = 0 - UNARY_OPERATOR = 1 - BINARY_OPERATOR = 2 - SUBSCRIPT_COLON = 3 - SUBSCRIPT_BRACKET = 4 - DEFAULT_OR_NAMED_ASSIGN = 5 - VARARGS_STAR = 6 - KWARGS_STAR_STAR = 7 - ASSIGN_OPERATOR = 8 - DICTIONARY_KEY = 9 - DICTIONARY_VALUE = 10 - DICT_SET_GENERATOR = 11 - COMP_FOR = 12 - COMP_IF = 13 - DEFAULT_OR_NAMED_ASSIGN_ARG_LIST = 14 - FUNC_DEF = 15 + if align_style in ('FIXED', 'VALIGN-RIGHT'): + if spaces > 0: + return '\t' * int((spaces + tab_width - 1) / tab_width) + return '' + return ' ' * spaces class FormatToken(object): - """A wrapper around pytree Leaf nodes. + """Enhanced token information for formatting. This represents the token plus additional information useful for reformatting the code. Attributes: - next_token: The token in the unwrapped line after this token or None if this - is the last token in the unwrapped line. - previous_token: The token in the unwrapped line before this token or None if - this is the first token in the unwrapped line. + node: The original token node. + next_token: The token in the logical line after this token or None if this + is the last token in the logical line. + previous_token: The token in the logical line before this token or None if + this is the first token in the logical line. matching_bracket: If a bracket token ('[', '{', or '(') the matching bracket. + parameters: If this and its following tokens make up a parameter list, then + this is a list of those parameters. + container_opening: If the object is in a container, this points to its + opening bracket. + container_elements: If this is the start of a container, a list of the + elements in the container. whitespace_prefix: The prefix for the whitespace. spaces_required_before: The number of spaces required before a token. This is a lower-bound for the formatter and not a hard requirement. For @@ -73,34 +75,64 @@ class FormatToken(object): formatter won't place n spaces before all comments. Only those that are moved to the end of a line of code. The formatter may use different spacing when appropriate. - can_break_before: True if we're allowed to break before this token. - must_break_before: True if we're required to break before this token. - total_length: The total length of the unwrapped line up to and including + total_length: The total length of the logical line up to and including whitespace and this token. However, this doesn't include the initial indentation amount. split_penalty: The penalty for splitting the line before this token. + can_break_before: True if we're allowed to break before this token. + must_break_before: True if we're required to break before this token. + newlines: The number of newlines needed before this token. """ - def __init__(self, node): + def __init__(self, node, name): """Constructor. Arguments: node: (pytree.Leaf) The node that's being wrapped. + name: (string) The name of the node. """ self.node = node + self.name = name + self.type = node.type + self.column = node.column + self.lineno = node.lineno + self.value = node.value + + if self.is_continuation: + self.value = node.value.rstrip() + self.next_token = None self.previous_token = None self.matching_bracket = None + self.parameters = [] + self.container_opening = None + self.container_elements = [] self.whitespace_prefix = '' - self.can_break_before = False - self.must_break_before = False - self.total_length = 0 # TODO(morbo): Think up a better name. + self.total_length = 0 self.split_penalty = 0 + self.can_break_before = False + self.must_break_before = pytree_utils.GetNodeAnnotation( + node, pytree_utils.Annotation.MUST_SPLIT, default=False) + self.newlines = pytree_utils.GetNodeAnnotation( + node, pytree_utils.Annotation.NEWLINES) + self.spaces_required_before = 0 if self.is_comment: self.spaces_required_before = style.Get('SPACES_BEFORE_COMMENT') - else: - self.spaces_required_before = 0 + + stypes = pytree_utils.GetNodeAnnotation(node, + pytree_utils.Annotation.SUBTYPE) + self.subtypes = {subtypes.NONE} if not stypes else stypes + self.is_pseudo = hasattr(node, 'is_pseudo') and node.is_pseudo + + @property + def formatted_whitespace_prefix(self): + if style.Get('INDENT_BLANK_LINES'): + without_newlines = self.whitespace_prefix.lstrip('\n') + height = len(self.whitespace_prefix) - len(without_newlines) + if height: + return ('\n' + without_newlines) * height + return self.whitespace_prefix def AddWhitespacePrefix(self, newlines_before, spaces=0, indent_level=0): """Register a token's whitespace prefix. @@ -112,48 +144,63 @@ def AddWhitespacePrefix(self, newlines_before, spaces=0, indent_level=0): spaces: (int) The number of spaces to place before the token. indent_level: (int) The indentation level. """ - indent_char = '\t' if style.Get('USE_TABS') else ' ' - indent_before = ( - indent_char * indent_level * style.Get('INDENT_WIDTH') + ' ' * spaces) + if style.Get('USE_TABS'): + if newlines_before > 0: + indent_before = '\t' * indent_level + _TabbedContinuationAlignPadding( + spaces, style.Get('CONTINUATION_ALIGN_STYLE'), + style.Get('INDENT_WIDTH')) + else: + indent_before = '\t' * indent_level + ' ' * spaces + else: + indent_before = (' ' * indent_level * style.Get('INDENT_WIDTH') + + ' ' * spaces) if self.is_comment: comment_lines = [s.lstrip() for s in self.value.splitlines()] - self.node.value = ('\n' + indent_before).join(comment_lines) + self.value = ('\n' + indent_before).join(comment_lines) + + # Update our own value since we are changing node value + self.value = self.value if not self.whitespace_prefix: - self.whitespace_prefix = ( - '\n' * (self.newlines or newlines_before) + indent_before) + self.whitespace_prefix = ('\n' * (self.newlines or newlines_before) + + indent_before) else: self.whitespace_prefix += indent_before def AdjustNewlinesBefore(self, newlines_before): """Change the number of newlines before this token.""" - self.whitespace_prefix = ( - '\n' * newlines_before + self.whitespace_prefix.lstrip('\n')) + self.whitespace_prefix = ('\n' * newlines_before + + self.whitespace_prefix.lstrip('\n')) def RetainHorizontalSpacing(self, first_column, depth): """Retains a token's horizontal spacing.""" previous = self.previous_token - if previous is None: + if not previous: return + if previous.is_pseudo: + previous = previous.previous_token + if not previous: + return + cur_lineno = self.lineno prev_lineno = previous.lineno if previous.is_multiline_string: prev_lineno += previous.value.count('\n') if (cur_lineno != prev_lineno or - (previous.is_pseudo_paren and previous.value != ')' and + (previous.is_pseudo and previous.value != ')' and cur_lineno != previous.previous_token.lineno)): self.spaces_required_before = ( self.column - first_column + depth * style.Get('INDENT_WIDTH')) return - cur_column = self.node.column - prev_column = previous.node.column + cur_column = self.column + prev_column = previous.column prev_len = len(previous.value) - if previous.is_pseudo_paren and previous.value == ')': + if previous.is_pseudo and previous.value == ')': prev_column -= 1 prev_len = 0 @@ -161,117 +208,145 @@ def RetainHorizontalSpacing(self, first_column, depth): prev_len = len(previous.value.split('\n')[-1]) if '\n' in previous.value: prev_column = 0 # Last line starts in column 0. + self.spaces_required_before = cur_column - (prev_column + prev_len) def OpensScope(self): - return self.value in pytree_utils.OPENING_BRACKETS + return self.value in _OPENING_BRACKETS def ClosesScope(self): - return self.value in pytree_utils.CLOSING_BRACKETS + return self.value in _CLOSING_BRACKETS + + def AddSubtype(self, subtype): + self.subtypes.add(subtype) def __repr__(self): - msg = 'FormatToken(name={0}, value={1}'.format(self.name, self.value) - msg += ', pseudo)' if self.is_pseudo_paren else ')' + msg = ('FormatToken(name={0}, value={1}, column={2}, lineno={3}, ' + 'splitpenalty={4}'.format( + 'DOCSTRING' if self.is_docstring else self.name, self.value, + self.column, self.lineno, self.split_penalty)) + msg += ', pseudo)' if self.is_pseudo else ')' return msg @property - def value(self): - if self.is_continuation: - return self.node.value.rstrip() - return self.node.value - - @property - @py3compat.lru_cache() def node_split_penalty(self): """Split penalty attached to the pytree node of this token.""" return pytree_utils.GetNodeAnnotation( self.node, pytree_utils.Annotation.SPLIT_PENALTY, default=0) @property - def newlines(self): - """The number of newlines needed before this token.""" - return pytree_utils.GetNodeAnnotation(self.node, - pytree_utils.Annotation.NEWLINES) - - @property - def must_split(self): - """Return true if the token requires a split before it.""" - return pytree_utils.GetNodeAnnotation(self.node, - pytree_utils.Annotation.MUST_SPLIT) - - @property - def column(self): - """The original column number of the node in the source.""" - return self.node.column - - @property - def lineno(self): - """The original line number of the node in the source.""" - return self.node.lineno + def is_binary_op(self): + """Token is a binary operator.""" + return subtypes.BINARY_OPERATOR in self.subtypes @property - @py3compat.lru_cache() - def subtypes(self): - """Extra type information for directing formatting.""" - value = pytree_utils.GetNodeAnnotation(self.node, - pytree_utils.Annotation.SUBTYPE) - return [Subtype.NONE] if value is None else value + @lru_cache() + def is_arithmetic_op(self): + """Token is an arithmetic operator.""" + return self.value in frozenset({ + '+', # Add + '-', # Subtract + '*', # Multiply + '@', # Matrix Multiply + '/', # Divide + '//', # Floor Divide + '%', # Modulo + '<<', # Left Shift + '>>', # Right Shift + '|', # Bitwise Or + '&', # Bitwise Add + '^', # Bitwise Xor + '**', # Power + }) @property - @py3compat.lru_cache() - def is_binary_op(self): - """Token is a binary operator.""" - return Subtype.BINARY_OPERATOR in self.subtypes + def is_simple_expr(self): + """Token is an operator in a simple expression.""" + return subtypes.SIMPLE_EXPRESSION in self.subtypes @property - @py3compat.lru_cache() - def name(self): - """A string representation of the node's name.""" - return pytree_utils.NodeName(self.node) + def is_subscript_colon(self): + """Token is a subscript colon.""" + return subtypes.SUBSCRIPT_COLON in self.subtypes @property def is_comment(self): - return self.node.type == token.COMMENT + return self.type == token.COMMENT @property def is_continuation(self): - return self.node.type == CONTINUATION + return self.type == CONTINUATION @property - @py3compat.lru_cache() + @lru_cache() def is_keyword(self): - return keyword.iskeyword(self.value) + return keyword.iskeyword( + self.value) or (self.value == 'match' and + type_repr(self.node.parent.type) == 'match_stmt') or ( + self.value == 'case' and + type_repr(self.node.parent.type) == 'case_block') @property - @py3compat.lru_cache() def is_name(self): - return self.node.type == token.NAME and not self.is_keyword + return self.type == token.NAME and not self.is_keyword @property def is_number(self): - return self.node.type == token.NUMBER + return self.type == token.NUMBER @property def is_string(self): - return self.node.type == token.STRING + return self.type == token.STRING @property - @py3compat.lru_cache() def is_multiline_string(self): - return (self.is_string and - re.match(r'^[uUbB]?[rR]?(?P"""|\'\'\').*(?P=delim)$', - self.value, re.DOTALL) is not None) + """Test if this string is a multiline string. + + Returns: + A multiline string always ends with triple quotes, so if it is a string + token, inspect the last 3 characters and return True if it is a triple + double or triple single quote mark. + """ + return self.is_string and self.value.endswith(('"""', "'''")) @property - @py3compat.lru_cache() def is_docstring(self): - return self.is_multiline_string and not self.node.prev_sibling + return self.is_string and self.previous_token is None @property - @py3compat.lru_cache() - def is_pseudo_paren(self): - return hasattr(self.node, 'is_pseudo') and self.node.is_pseudo + def is_pylint_comment(self): + return self.is_comment and re.match(r'#.*\bpylint:\s*(disable|enable)=', + self.value) @property - def is_pylint_comment(self): - return self.is_comment and re.match(r'#\s*\bpylint:\b', self.value) + def is_pytype_comment(self): + return self.is_comment and re.match(r'#.*\bpytype:\s*(disable|enable)=', + self.value) + + @property + def is_copybara_comment(self): + return self.is_comment and re.match( + r'#.*\bcopybara:\s*(strip|insert|replace)', self.value) + + @property + def is_assign(self): + return subtypes.ASSIGN_OPERATOR in self.subtypes + + @property + @lru_cache() + def is_augassign(self): + return self.value in frozenset({ + '+=', + '-=', + '*=', + '@=', + '/=', + '//=', + '%=', + '<<=', + '>>=', + '|=', + '&=', + '^=', + '**=', + }) diff --git a/yapf/yapflib/identify_container.py b/yapf/yapflib/identify_container.py new file mode 100644 index 000000000..43ba4b9ed --- /dev/null +++ b/yapf/yapflib/identify_container.py @@ -0,0 +1,69 @@ +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Identify containers for lib2to3 trees. + +This module identifies containers and the elements in them. Each element points +to the opening bracket and vice-versa. + + IdentifyContainers(): the main function exported by this module. +""" + +from yapf_third_party._ylib2to3.pgen2 import token as grammar_token + +from yapf.pytree import pytree_utils +from yapf.pytree import pytree_visitor + + +def IdentifyContainers(tree): + """Run the identify containers visitor over the tree, modifying it in place. + + Arguments: + tree: the top-level pytree node to annotate with subtypes. + """ + identify_containers = _IdentifyContainers() + identify_containers.Visit(tree) + + +class _IdentifyContainers(pytree_visitor.PyTreeVisitor): + """_IdentifyContainers - see file-level docstring for detailed description.""" + + def Visit_trailer(self, node): # pylint: disable=invalid-name + for child in node.children: + self.Visit(child) + + if len(node.children) != 3: + return + if node.children[0].type != grammar_token.LPAR: + return + + if pytree_utils.NodeName(node.children[1]) == 'arglist': + for child in node.children[1].children: + pytree_utils.SetOpeningBracket( + pytree_utils.FirstLeafNode(child), node.children[0]) + else: + pytree_utils.SetOpeningBracket( + pytree_utils.FirstLeafNode(node.children[1]), node.children[0]) + + def Visit_atom(self, node): # pylint: disable=invalid-name + for child in node.children: + self.Visit(child) + + if len(node.children) != 3: + return + if node.children[0].type != grammar_token.LPAR: + return + + for child in node.children[1].children: + pytree_utils.SetOpeningBracket( + pytree_utils.FirstLeafNode(child), node.children[0]) diff --git a/yapf/yapflib/line_joiner.py b/yapf/yapflib/line_joiner.py index e7a168440..f0acd2f37 100644 --- a/yapf/yapflib/line_joiner.py +++ b/yapf/yapflib/line_joiner.py @@ -1,4 +1,4 @@ -# Copyright 2015-2016 Google Inc. All Rights Reserved. +# Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Join unwrapped lines together. +"""Join logical lines together. Determine how many lines can be joined into one line. For instance, we could join these statements into one line: @@ -43,8 +43,8 @@ def CanMergeMultipleLines(lines, last_was_merged=False): """Determine if multiple lines can be joined into one. Arguments: - lines: (list of UnwrappedLine) This is a splice of UnwrappedLines from the - full code base. + lines: (list of LogicalLine) This is a splice of LogicalLines from the full + code base. last_was_merged: (bool) The last line was merged. Returns: @@ -91,7 +91,7 @@ def _CanMergeLineIntoIfStatement(lines, limit): 'continue', and 'break'. Arguments: - lines: (list of UnwrappedLine) The lines we are wanting to merge. + lines: (list of LogicalLine) The lines we are wanting to merge. limit: (int) The amount of space remaining on the line. Returns: diff --git a/yapf/yapflib/unwrapped_line.py b/yapf/yapflib/logical_line.py similarity index 50% rename from yapf/yapflib/unwrapped_line.py rename to yapf/yapflib/logical_line.py index 14eaa8e97..4433276f7 100644 --- a/yapf/yapflib/unwrapped_line.py +++ b/yapf/yapflib/logical_line.py @@ -1,4 +1,4 @@ -# Copyright 2015-2016 Google Inc. All Rights Reserved. +# Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,23 +11,25 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""UnwrappedLine primitive for formatting. +"""LogicalLine primitive for formatting. -An unwrapped line is the containing data structure produced by the parser. It -collects all nodes (stored in FormatToken objects) that could appear on a -single line if there were no line length restrictions. It's then used by the -parser to perform the wrapping required to comply with the style guide. +A logical line is the containing data structure produced by the parser. It +collects all nodes (stored in FormatToken objects) that could appear on a single +line if there were no line length restrictions. It's then used by the parser to +perform the wrapping required to comply with the style guide. """ +from yapf_third_party._ylib2to3.fixer_util import syms as python_symbols + +from yapf.pytree import pytree_utils +from yapf.pytree import split_penalty from yapf.yapflib import format_token -from yapf.yapflib import py3compat -from yapf.yapflib import pytree_utils -from yapf.yapflib import split_penalty from yapf.yapflib import style +from yapf.yapflib import subtypes -class UnwrappedLine(object): - """Represents a single unwrapped line in the output. +class LogicalLine(object): + """Represents a single logical line in the output. Attributes: depth: indentation depth of this line. This is just a numeric value used to @@ -38,7 +40,7 @@ class UnwrappedLine(object): def __init__(self, depth, tokens=None): """Constructor. - Creates a new unwrapped line with the given depth an initial list of tokens. + Creates a new logical line with the given depth an initial list of tokens. Constructs the doubly-linked lists for format tokens using their built-in next_token and previous_token attributes. @@ -60,7 +62,7 @@ def __init__(self, depth, tokens=None): def CalculateFormattingInformation(self): """Calculate the split penalty and total length for the tokens.""" # Say that the first token in the line should have a space before it. This - # means only that if this unwrapped line is joined with a predecessor line, + # means only that if this logical line is joined with a predecessor line, # then there will be a space between them. self.first.spaces_required_before = 1 self.first.total_length = len(self.first.value) @@ -69,22 +71,58 @@ def CalculateFormattingInformation(self): prev_length = self.first.total_length for token in self._tokens[1:]: if (token.spaces_required_before == 0 and - _SpaceRequiredBetween(prev_token, token)): + _SpaceRequiredBetween(prev_token, token, self.disable)): token.spaces_required_before = 1 - token.total_length = ( - prev_length + len(token.value) + token.spaces_required_before) + tok_len = len(token.value) if not token.is_pseudo else 0 + + spaces_required_before = token.spaces_required_before + if isinstance(spaces_required_before, list): + assert token.is_comment, token + + # If here, we are looking at a comment token that appears on a line + # with other tokens (but because it is a comment, it is always the last + # token). Rather than specifying the actual number of spaces here, + # hard code a value of 0 and then set it later. This logic only works + # because this comment token is guaranteed to be the last token in the + # list. + spaces_required_before = 0 + + token.total_length = prev_length + tok_len + spaces_required_before # The split penalty has to be computed before {must|can}_break_before, # because these may use it for their decision. token.split_penalty += _SplitPenalty(prev_token, token) token.must_break_before = _MustBreakBefore(prev_token, token) - token.can_break_before = (token.must_break_before or - _CanBreakBefore(prev_token, token)) + token.can_break_before = ( + token.must_break_before or _CanBreakBefore(prev_token, token)) prev_length = token.total_length prev_token = token + def Split(self): + """Split the line at semicolons.""" + if not self.has_semicolon or self.disable: + return [self] + + llines = [] + lline = LogicalLine(self.depth) + for tok in self._tokens: + if tok.value == ';': + llines.append(lline) + lline = LogicalLine(self.depth) + else: + lline.AppendToken(tok) + + if lline.tokens: + llines.append(lline) + + for lline in llines: + lline.first.previous_token = None + lline.last.next_token = None + + return llines + ############################################################################ # Token Access and Manipulation Methods # ############################################################################ @@ -96,16 +134,6 @@ def AppendToken(self, token): self.last.next_token = token self._tokens.append(token) - def AppendNode(self, node): - """Convenience method to append a pytree node directly. - - Wraps the node with a FormatToken. - - Arguments: - node: the node to append - """ - self.AppendToken(format_token.FormatToken(node)) - @property def first(self): """Returns the first non-whitespace token.""" @@ -130,7 +158,7 @@ def AsCode(self, indent_per_depth=2): have spaces around them, for example). Arguments: - indent_per_depth: how much spaces to indend per depth level. + indent_per_depth: how much spaces to indent per depth level. Returns: A string representing the line as code. @@ -143,10 +171,10 @@ def __str__(self): # pragma: no cover return self.AsCode() def __repr__(self): # pragma: no cover - tokens_repr = ','.join(['{0}({1!r})'.format(tok.name, tok.value) - for tok in self._tokens]) - return 'UnwrappedLine(depth={0}, tokens=[{1}])'.format(self.depth, - tokens_repr) + tokens_repr = ','.join( + '{0}({1!r})'.format(tok.name, tok.value) for tok in self._tokens) + return 'LogicalLine(depth={0}, tokens=[{1}])'.format( + self.depth, tokens_repr) ############################################################################ # Properties # @@ -165,36 +193,99 @@ def tokens(self): @property def lineno(self): - """Return the line number of this unwrapped line. + """Return the line number of this logical line. Returns: - The line number of the first token in this unwrapped line. + The line number of the first token in this logical line. """ return self.first.lineno + @property + def start(self): + """The start of the logical line. + + Returns: + A tuple of the starting line number and column. + """ + return (self.first.lineno, self.first.column) + + @property + def end(self): + """The end of the logical line. + + Returns: + A tuple of the ending line number and column. + """ + return (self.last.lineno, self.last.column + len(self.last.value)) + @property def is_comment(self): return self.first.is_comment + @property + def has_semicolon(self): + return any(tok.value == ';' for tok in self._tokens) + def _IsIdNumberStringToken(tok): return tok.is_keyword or tok.is_name or tok.is_number or tok.is_string def _IsUnaryOperator(tok): - return format_token.Subtype.UNARY_OPERATOR in tok.subtypes + return subtypes.UNARY_OPERATOR in tok.subtypes + + +def _HasPrecedence(tok): + """Whether a binary operation has precedence within its context.""" + node = tok.node + + # We let ancestor be the statement surrounding the operation that tok is the + # operator in. + ancestor = node.parent.parent + + while ancestor is not None: + # Search through the ancestor nodes in the parse tree for operators with + # lower precedence. + predecessor_type = pytree_utils.NodeName(ancestor) + if predecessor_type in ['arith_expr', 'term']: + # An ancestor "arith_expr" or "term" means we have found an operator + # with lower precedence than our tok. + return True + if predecessor_type != 'atom': + # We understand the context to look for precedence within as an + # arbitrary nesting of "arith_expr", "term", and "atom" nodes. If we + # leave this context we have not found a lower precedence operator. + return False + # Under normal usage we expect a complete parse tree to be available and + # we will return before we get an AttributeError from the root. + ancestor = ancestor.parent + +def _PriorityIndicatingNoSpace(tok): + """Whether to remove spaces around an operator due to precedence.""" + if not tok.is_arithmetic_op or not tok.is_simple_expr: + # Limit space removal to highest priority arithmetic operators + return False + return _HasPrecedence(tok) + + +def _IsSubscriptColonAndValuePair(token1, token2): + return (token1.is_number or token1.is_name) and token2.is_subscript_colon -def _SpaceRequiredBetween(left, right): + +def _SpaceRequiredBetween(left, right, is_line_disabled): """Return True if a space is required between the left and right token.""" lval = left.value rval = right.value - if (left.is_pseudo_paren and _IsIdNumberStringToken(right) and + if (left.is_pseudo and _IsIdNumberStringToken(right) and left.previous_token and _IsIdNumberStringToken(left.previous_token)): # Space between keyword... tokens and pseudo parens. return True - if left.is_pseudo_paren or right.is_pseudo_paren: - # The pseudo-parens shouldn't affect spacing. + if left.is_pseudo or right.is_pseudo: + # There should be a space after the ':' in a dictionary. + if left.OpensScope(): + return True + # The closing pseudo-paren shouldn't affect spacing. return False if left.is_continuation or right.is_continuation: # The continuation node's value has all of the spaces it needs. @@ -208,6 +299,17 @@ def _SpaceRequiredBetween(left, right): if lval == ',' and rval == ':': # We do want a space between a comma and colon. return True + if style.Get('SPACE_INSIDE_BRACKETS'): + # Supersede the "no space before a colon or comma" check. + if left.OpensScope() and rval == ':': + return True + if right.ClosesScope() and lval == ':': + return True + if (style.Get('SPACES_AROUND_SUBSCRIPT_COLON') and + (_IsSubscriptColonAndValuePair(left, right) or + _IsSubscriptColonAndValuePair(right, left))): + # Supersede the "never want a space before a colon or comma" check. + return True if rval in ':,': # Otherwise, we never want a space before a colon or comma. return False @@ -223,16 +325,48 @@ def _SpaceRequiredBetween(left, right): if lval == '.' and rval == 'import': # Space after the '.' in an import statement. return True + if (lval == '=' and rval in {'.', ',,,'} and + subtypes.DEFAULT_OR_NAMED_ASSIGN not in left.subtypes): + # Space between equal and '.' as in "X = ...". + return True + if lval == ':' and rval in {'.', '...'}: + # Space between : and ... + return True if ((right.is_keyword or right.is_name) and (left.is_keyword or left.is_name)): # Don't merge two keywords/identifiers. return True - if left.is_string and rval not in '[)]}.': - # A string followed by something other than a subscript, closing bracket, - # or dot should have a space after it. + if (subtypes.SUBSCRIPT_COLON in left.subtypes or + subtypes.SUBSCRIPT_COLON in right.subtypes): + # A subscript shouldn't have spaces separating its colons. + return False + if (subtypes.TYPED_NAME in left.subtypes or + subtypes.TYPED_NAME in right.subtypes): + # A typed argument should have a space after the colon. + return True + if left.is_string: + if (rval == '=' and + subtypes.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST in right.subtypes): + # If there is a type hint, then we don't want to add a space between the + # equal sign and the hint. + return False + if rval not in '[)]}.' and not right.is_binary_op: + # A string followed by something other than a subscript, closing bracket, + # dot, or a binary op should have a space after it. + return True + if right.ClosesScope(): + # A string followed by closing brackets should have a space after it + # depending on SPACE_INSIDE_BRACKETS. A string followed by opening + # brackets, however, should not. + return style.Get('SPACE_INSIDE_BRACKETS') + if subtypes.SUBSCRIPT_BRACKET in right.subtypes: + # It's legal to do this in Python: 'hello'[a] + return False + if left.is_binary_op and lval != '**' and _IsUnaryOperator(right): + # Space between the binary operator and the unary operator. return True - if left.is_binary_op and _IsUnaryOperator(right): - # Space between the binary opertor and the unary operator. + if left.is_keyword and _IsUnaryOperator(right): + # Handle things like "not -3 < x". return True if _IsUnaryOperator(left) and _IsUnaryOperator(right): # No space between two unary operators. @@ -241,81 +375,120 @@ def _SpaceRequiredBetween(left, right): if lval == '**' or rval == '**': # Space around the "power" operator. return style.Get('SPACES_AROUND_POWER_OPERATOR') - # Enforce spaces around binary operators. - return True + # Enforce spaces around binary operators except the blocked ones. + block_list = style.Get('NO_SPACES_AROUND_SELECTED_BINARY_OPERATORS') + if lval in block_list or rval in block_list: + return False + if style.Get('ARITHMETIC_PRECEDENCE_INDICATION'): + if _PriorityIndicatingNoSpace(left) or _PriorityIndicatingNoSpace(right): + return False + else: + return True + else: + return True if (_IsUnaryOperator(left) and lval != 'not' and (right.is_name or right.is_number or rval == '(')): # The previous token was a unary op. No space is desired between it and # the current token. return False - if (format_token.Subtype.SUBSCRIPT_COLON in left.subtypes or - format_token.Subtype.SUBSCRIPT_COLON in right.subtypes): - # A subscript shouldn't have spaces separating its colons. - return False - if (format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN in left.subtypes or - format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN in right.subtypes): + if (subtypes.DEFAULT_OR_NAMED_ASSIGN in left.subtypes and + subtypes.TYPED_NAME not in right.subtypes): # A named argument or default parameter shouldn't have spaces around it. + return style.Get('SPACES_AROUND_DEFAULT_OR_NAMED_ASSIGN') + if (subtypes.DEFAULT_OR_NAMED_ASSIGN in right.subtypes and + subtypes.TYPED_NAME not in left.subtypes): + # A named argument or default parameter shouldn't have spaces around it. + return style.Get('SPACES_AROUND_DEFAULT_OR_NAMED_ASSIGN') + if (subtypes.VARARGS_LIST in left.subtypes or + subtypes.VARARGS_LIST in right.subtypes): return False - if (format_token.Subtype.VARARGS_STAR in left.subtypes or - format_token.Subtype.KWARGS_STAR_STAR in left.subtypes): + if (subtypes.VARARGS_STAR in left.subtypes or + subtypes.KWARGS_STAR_STAR in left.subtypes): # Don't add a space after a vararg's star or a keyword's star-star. return False - if lval == '@': + if lval == '@' and subtypes.DECORATOR in left.subtypes: # Decorators shouldn't be separated from the 'at' sign. return False + if left.is_keyword and rval == '.': + # Add space between keywords and dots. + return lval not in {'None', 'print'} + if lval == '.' and right.is_keyword: + # Add space between keywords and dots. + return rval not in {'None', 'print'} if lval == '.' or rval == '.': # Don't place spaces between dots. return False if ((lval == '(' and rval == ')') or (lval == '[' and rval == ']') or (lval == '{' and rval == '}')): - # Empty objects shouldn't be separted by spaces. - return False - if (lval in pytree_utils.OPENING_BRACKETS and - rval in pytree_utils.OPENING_BRACKETS): - # Nested objects' opening brackets shouldn't be separated. - return False - if (lval in pytree_utils.CLOSING_BRACKETS and - rval in pytree_utils.CLOSING_BRACKETS): - # Nested objects' closing brackets shouldn't be separated. - return False - if lval in pytree_utils.CLOSING_BRACKETS and rval in '([': + # Empty objects shouldn't be separated by spaces. + return False + if not is_line_disabled and (left.OpensScope() or right.ClosesScope()): + if (style.GetOrDefault('SPACES_AROUND_DICT_DELIMITERS', False) and ( + (lval == '{' and _IsDictListTupleDelimiterTok(left, is_opening=True)) or + (rval == '}' and + _IsDictListTupleDelimiterTok(right, is_opening=False)))): + return True + if (style.GetOrDefault('SPACES_AROUND_LIST_DELIMITERS', False) and ( + (lval == '[' and _IsDictListTupleDelimiterTok(left, is_opening=True)) or + (rval == ']' and + _IsDictListTupleDelimiterTok(right, is_opening=False)))): + return True + if (style.GetOrDefault('SPACES_AROUND_TUPLE_DELIMITERS', False) and ( + (lval == '(' and _IsDictListTupleDelimiterTok(left, is_opening=True)) or + (rval == ')' and + _IsDictListTupleDelimiterTok(right, is_opening=False)))): + return True + if left.OpensScope() and right.OpensScope(): + # Nested objects' opening brackets shouldn't be separated, unless enabled + # by SPACE_INSIDE_BRACKETS. + return style.Get('SPACE_INSIDE_BRACKETS') + if left.ClosesScope() and right.ClosesScope(): + # Nested objects' closing brackets shouldn't be separated, unless enabled + # by SPACE_INSIDE_BRACKETS. + return style.Get('SPACE_INSIDE_BRACKETS') + if left.ClosesScope() and rval in '([': # A call, set, dictionary, or subscript that has a call or subscript after # it shouldn't have a space between them. return False - if lval in pytree_utils.OPENING_BRACKETS and _IsIdNumberStringToken(right): - # Don't separate the opening bracket from the first item. - return False + if left.OpensScope() and _IsIdNumberStringToken(right): + # Don't separate the opening bracket from the first item, unless enabled + # by SPACE_INSIDE_BRACKETS. + return style.Get('SPACE_INSIDE_BRACKETS') if left.is_name and rval in '([': # Don't separate a call or array access from the name. return False - if rval in pytree_utils.CLOSING_BRACKETS: - # Don't separate the closing bracket from the last item. + if right.ClosesScope(): + # Don't separate the closing bracket from the last item, unless enabled + # by SPACE_INSIDE_BRACKETS. # FIXME(morbo): This might be too permissive. - return False + return style.Get('SPACE_INSIDE_BRACKETS') if lval == 'print' and rval == '(': # Special support for the 'print' function. return False - if lval in pytree_utils.OPENING_BRACKETS and _IsUnaryOperator(right): - # Don't separate a unary operator from the opening bracket. - return False - if (lval in pytree_utils.OPENING_BRACKETS and - (format_token.Subtype.VARARGS_STAR in right.subtypes or - format_token.Subtype.KWARGS_STAR_STAR in right.subtypes)): - # Don't separate a '*' or '**' from the opening bracket. - return False + if left.OpensScope() and _IsUnaryOperator(right): + # Don't separate a unary operator from the opening bracket, unless enabled + # by SPACE_INSIDE_BRACKETS. + return style.Get('SPACE_INSIDE_BRACKETS') + if (left.OpensScope() and (subtypes.VARARGS_STAR in right.subtypes or + subtypes.KWARGS_STAR_STAR in right.subtypes)): + # Don't separate a '*' or '**' from the opening bracket, unless enabled + # by SPACE_INSIDE_BRACKETS. + return style.Get('SPACE_INSIDE_BRACKETS') if rval == ';': # Avoid spaces before a semicolon. (Why is there a semicolon?!) return False if lval == '(' and rval == 'await': # Special support for the 'await' keyword. Don't separate the 'await' - # keyword from an opening paren. - return False + # keyword from an opening paren, unless enabled by SPACE_INSIDE_BRACKETS. + return style.Get('SPACE_INSIDE_BRACKETS') return True def _MustBreakBefore(prev_token, cur_token): """Return True if a line break is required before the current token.""" - if prev_token.is_comment: + if prev_token.is_comment or (prev_token.previous_token and + prev_token.is_pseudo and + prev_token.previous_token.is_comment): # Must break if the previous token was a comment. return True if (cur_token.is_string and prev_token.is_string and @@ -324,21 +497,19 @@ def _MustBreakBefore(prev_token, cur_token): # reasonable assumption, because otherwise they should have written them # all on the same line, or with a '+'. return True - return pytree_utils.GetNodeAnnotation( - cur_token.node, pytree_utils.Annotation.MUST_SPLIT, default=False) + return cur_token.must_break_before def _CanBreakBefore(prev_token, cur_token): """Return True if a line break may occur before the current token.""" pval = prev_token.value cval = cur_token.value - if py3compat.PY3: - if pval == 'yield' and cval == 'from': - # Don't break before a yield argument. - return False - if pval in {'async', 'await'} and cval in {'def', 'with', 'for'}: - # Don't break after sync keywords. - return False + if pval == 'yield' and cval == 'from': + # Don't break before a yield argument. + return False + if pval in {'async', 'await'} and cval in {'def', 'with', 'for'}: + # Don't break after sync keywords. + return False if cur_token.split_penalty >= split_penalty.UNBREAKABLE: return False if pval == '@': @@ -356,15 +527,16 @@ def _CanBreakBefore(prev_token, cur_token): if prev_token.is_name and cval == '[': # Don't break in the middle of an array dereference. return False - if prev_token.is_name and cval == '.': - # Don't break before the '.' in a dotted name. - return False if cur_token.is_comment and prev_token.lineno == cur_token.lineno: # Don't break a comment at the end of the line. return False - if format_token.Subtype.UNARY_OPERATOR in prev_token.subtypes: + if subtypes.UNARY_OPERATOR in prev_token.subtypes: # Don't break after a unary token. return False + if not style.Get('ALLOW_SPLIT_BEFORE_DEFAULT_OR_NAMED_ASSIGNS'): + if (subtypes.DEFAULT_OR_NAMED_ASSIGN in cur_token.subtypes or + subtypes.DEFAULT_OR_NAMED_ASSIGN in prev_token.subtypes): + return False return True @@ -384,24 +556,51 @@ def IsSurroundedByBrackets(tok): if previous_token.value == '(': if paren_count == 0: - return True + return previous_token paren_count += 1 elif previous_token.value == '{': if brace_count == 0: - return True + return previous_token brace_count += 1 elif previous_token.value == '[': if sq_bracket_count == 0: - return True + return previous_token sq_bracket_count += 1 previous_token = previous_token.previous_token - return False + return None + + +def _IsDictListTupleDelimiterTok(tok, is_opening): + assert tok + + if tok.matching_bracket is None: + return False + + if is_opening: + open_tok = tok + close_tok = tok.matching_bracket + else: + open_tok = tok.matching_bracket + close_tok = tok + + # There must be something in between the tokens + if open_tok.next_token == close_tok: + return False + + assert open_tok.next_token.node + assert open_tok.next_token.node.parent + + return open_tok.next_token.node.parent.type in [ + python_symbols.dictsetmaker, + python_symbols.listmaker, + python_symbols.testlist_gexp, + ] _LOGICAL_OPERATORS = frozenset({'and', 'or'}) _BITWISE_OPERATORS = frozenset({'&', '|', '^'}) -_TERM_OPERATORS = frozenset({'*', '/', '%', '//'}) +_ARITHMETIC_OPERATORS = frozenset({'+', '-', '*', '/', '%', '//', '@'}) def _SplitPenalty(prev_token, cur_token): @@ -440,21 +639,20 @@ def _SplitPenalty(prev_token, cur_token): if cval in _BITWISE_OPERATORS: return style.Get('SPLIT_PENALTY_BITWISE_OPERATOR') - if (format_token.Subtype.COMP_FOR in cur_token.subtypes or - format_token.Subtype.COMP_IF in cur_token.subtypes): + if (subtypes.COMP_FOR in cur_token.subtypes or + subtypes.COMP_IF in cur_token.subtypes): # We don't mind breaking before the 'for' or 'if' of a list comprehension. return 0 - if format_token.Subtype.UNARY_OPERATOR in prev_token.subtypes: + if subtypes.UNARY_OPERATOR in prev_token.subtypes: # Try not to break after a unary operator. return style.Get('SPLIT_PENALTY_AFTER_UNARY_OPERATOR') if pval == ',': # Breaking after a comma is fine, if need be. return 0 - if prev_token.is_binary_op: - # We would rather not split after an equality operator. - return 20 - if (format_token.Subtype.VARARGS_STAR in prev_token.subtypes or - format_token.Subtype.KWARGS_STAR_STAR in prev_token.subtypes): + if pval == '**' or cval == '**': + return split_penalty.STRONGLY_CONNECTED + if (subtypes.VARARGS_STAR in prev_token.subtypes or + subtypes.KWARGS_STAR_STAR in prev_token.subtypes): # Don't split after a varargs * or kwargs **. return split_penalty.UNBREAKABLE if prev_token.OpensScope() and cval != '(': @@ -466,8 +664,8 @@ def _SplitPenalty(prev_token, cur_token): if cval == '=': # Don't split before an assignment. return split_penalty.UNBREAKABLE - if (format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN in prev_token.subtypes or - format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN in cur_token.subtypes): + if (subtypes.DEFAULT_OR_NAMED_ASSIGN in prev_token.subtypes or + subtypes.DEFAULT_OR_NAMED_ASSIGN in cur_token.subtypes): # Don't break before or after an default or named assignment. return split_penalty.UNBREAKABLE if cval == '==': @@ -476,6 +674,4 @@ def _SplitPenalty(prev_token, cur_token): if cur_token.ClosesScope(): # Give a slight penalty for splitting before the closing scope. return 100 - if pval in _TERM_OPERATORS or cval in _TERM_OPERATORS: - return 50 return 0 diff --git a/yapf/yapflib/object_state.py b/yapf/yapflib/object_state.py new file mode 100644 index 000000000..cb8c51b05 --- /dev/null +++ b/yapf/yapflib/object_state.py @@ -0,0 +1,229 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Represents the state of Python objects being formatted. + +Objects (e.g., list comprehensions, dictionaries, etc.) have specific +requirements on how they're formatted. These state objects keep track of these +requirements. +""" + +from functools import lru_cache + +from yapf.yapflib import style +from yapf.yapflib import subtypes + + +class ComprehensionState(object): + """Maintains the state of list comprehension formatting decisions. + + A stack of ComprehensionState objects are kept to ensure that list + comprehensions are wrapped with well-defined rules. + + Attributes: + expr_token: The first token in the comprehension. + for_token: The first 'for' token of the comprehension. + opening_bracket: The opening bracket of the list comprehension. + closing_bracket: The closing bracket of the list comprehension. + has_split_at_for: Whether there is a newline immediately before the + for_token. + has_interior_split: Whether there is a newline within the comprehension. + That is, a split somewhere after expr_token or before closing_bracket. + """ + + def __init__(self, expr_token): + self.expr_token = expr_token + self.for_token = None + self.has_split_at_for = False + self.has_interior_split = False + + def HasTrivialExpr(self): + """Returns whether the comp_expr is "trivial" i.e. is a single token.""" + return self.expr_token.next_token.value == 'for' + + @property + def opening_bracket(self): + return self.expr_token.previous_token + + @property + def closing_bracket(self): + return self.opening_bracket.matching_bracket + + def Clone(self): + clone = ComprehensionState(self.expr_token) + clone.for_token = self.for_token + clone.has_split_at_for = self.has_split_at_for + clone.has_interior_split = self.has_interior_split + return clone + + def __repr__(self): + return ('[opening_bracket::%s, for_token::%s, has_split_at_for::%s,' + ' has_interior_split::%s, has_trivial_expr::%s]' % + (self.opening_bracket, self.for_token, self.has_split_at_for, + self.has_interior_split, self.HasTrivialExpr())) + + def __eq__(self, other): + return hash(self) == hash(other) + + def __ne__(self, other): + return not self == other + + def __hash__(self, *args, **kwargs): + return hash((self.expr_token, self.for_token, self.has_split_at_for, + self.has_interior_split)) + + +class ParameterListState(object): + """Maintains the state of function parameter list formatting decisions. + + Attributes: + opening_bracket: The opening bracket of the parameter list. + closing_bracket: The closing bracket of the parameter list. + has_typed_return: True if the function definition has a typed return. + ends_in_comma: True if the parameter list ends in a comma. + last_token: Returns the last token of the function declaration. + has_default_values: True if the parameters have default values. + has_split_before_first_param: Whether there is a newline before the first + parameter. + opening_column: The position of the opening parameter before a newline. + parameters: A list of parameter objects (Parameter). + split_before_closing_bracket: Split before the closing bracket. Sometimes + needed if the indentation would collide. + """ + + def __init__(self, opening_bracket, newline, opening_column): + self.opening_bracket = opening_bracket + self.has_split_before_first_param = newline + self.opening_column = opening_column + self.parameters = opening_bracket.parameters + self.split_before_closing_bracket = False + + @property + def closing_bracket(self): + return self.opening_bracket.matching_bracket + + @property + def has_typed_return(self): + return self.closing_bracket.next_token.value == '->' + + @property + @lru_cache() + def has_default_values(self): + return any(param.has_default_value for param in self.parameters) + + @property + @lru_cache() + def ends_in_comma(self): + if not self.parameters: + return False + return self.parameters[-1].last_token.next_token.value == ',' + + @property + @lru_cache() + def last_token(self): + token = self.opening_bracket.matching_bracket + while not token.is_comment and token.next_token: + token = token.next_token + return token + + @lru_cache() + def LastParamFitsOnLine(self, indent): + """Return true if the last parameter fits on a single line.""" + if not self.has_typed_return: + return False + if not self.parameters: + return True + total_length = self.last_token.total_length + last_param = self.parameters[-1].first_token + total_length -= last_param.total_length - len(last_param.value) + return total_length + indent <= style.Get('COLUMN_LIMIT') + + @lru_cache() + def SplitBeforeClosingBracket(self, indent): + """Return true if there's a split before the closing bracket.""" + if style.Get('DEDENT_CLOSING_BRACKETS'): + return True + if self.ends_in_comma: + return True + if not self.parameters: + return False + total_length = self.last_token.total_length + last_param = self.parameters[-1].first_token + total_length -= last_param.total_length - len(last_param.value) + return total_length + indent > style.Get('COLUMN_LIMIT') + + def Clone(self): + clone = ParameterListState(self.opening_bracket, + self.has_split_before_first_param, + self.opening_column) + clone.split_before_closing_bracket = self.split_before_closing_bracket + clone.parameters = [param.Clone() for param in self.parameters] + return clone + + def __repr__(self): + return ('[opening_bracket::%s, has_split_before_first_param::%s, ' + 'opening_column::%d]' % + (self.opening_bracket, self.has_split_before_first_param, + self.opening_column)) + + def __eq__(self, other): + return hash(self) == hash(other) + + def __ne__(self, other): + return not self == other + + def __hash__(self, *args, **kwargs): + return hash( + (self.opening_bracket, self.has_split_before_first_param, + self.opening_column, (hash(param) for param in self.parameters))) + + +class Parameter(object): + """A parameter in a parameter list. + + Attributes: + first_token: (format_token.FormatToken) First token of parameter. + last_token: (format_token.FormatToken) Last token of parameter. + has_default_value: (boolean) True if the parameter has a default value + """ + + def __init__(self, first_token, last_token): + self.first_token = first_token + self.last_token = last_token + + @property + @lru_cache() + def has_default_value(self): + """Returns true if the parameter has a default value.""" + tok = self.first_token + while tok != self.last_token: + if subtypes.DEFAULT_OR_NAMED_ASSIGN in tok.subtypes: + return True + tok = tok.matching_bracket if tok.OpensScope() else tok.next_token + return False + + def Clone(self): + return Parameter(self.first_token, self.last_token) + + def __repr__(self): + return '[first_token::%s, last_token:%s]' % (self.first_token, + self.last_token) + + def __eq__(self, other): + return hash(self) == hash(other) + + def __ne__(self, other): + return not self == other + + def __hash__(self, *args, **kwargs): + return hash((self.first_token, self.last_token)) diff --git a/yapf/yapflib/py3compat.py b/yapf/yapflib/py3compat.py deleted file mode 100644 index c19cab0b8..000000000 --- a/yapf/yapflib/py3compat.py +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright 2015-2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Utilities for Python2 / Python3 compatibility.""" - -import io -import sys - -PY3 = sys.version_info[0] == 3 - -if PY3: - StringIO = io.StringIO - BytesIO = io.BytesIO - - import codecs - open_with_encoding = codecs.open - - import functools - lru_cache = functools.lru_cache - - range = range - ifilter = filter - raw_input = input - - import configparser - - # Mappings from strings to booleans (such as '1' to True, 'false' to False, - # etc.) - CONFIGPARSER_BOOLEAN_STATES = configparser.ConfigParser.BOOLEAN_STATES -else: - import __builtin__ - import cStringIO - StringIO = BytesIO = cStringIO.StringIO - - open_with_encoding = io.open - - # Python 2.7 doesn't have a native LRU cache, so do nothing. - def lru_cache(maxsize=128, typed=False): - - def fake_wrapper(user_function): - return user_function - - return fake_wrapper - - range = xrange - - from itertools import ifilter - raw_input = raw_input - - import ConfigParser as configparser - CONFIGPARSER_BOOLEAN_STATES = configparser.ConfigParser._boolean_states # pylint: disable=protected-access - - -def EncodeAndWriteToStdout(s, encoding): - """Encode the given string and emit to stdout. - - The string may contain non-ascii characters. This is a problem when stdout is - redirected, because then Python doesn't know the encoding and we may get a - UnicodeEncodeError. - - Arguments: - s: (string) The string to encode. - encoding: (string) The encoding of the string. - """ - if PY3: - sys.stdout.buffer.write(codecs.encode(s, encoding)) - else: - sys.stdout.write(s.encode(encoding)) - - -def unicode(s): - """Force conversion of s to unicode.""" - if PY3: - return s - else: - return __builtin__.unicode(s, 'utf-8') - - -# In Python 3.2+, readfp is deprecated in favor of read_file, which doesn't -# exist in Python 2 yet. To avoid deprecation warnings, subclass ConfigParser to -# fix this - now read_file works across all Python versions we care about. -class ConfigParser(configparser.ConfigParser): - if not PY3: - - def read_file(self, fp, source=None): - self.readfp(fp, filename=source) diff --git a/yapf/yapflib/reformatter.py b/yapf/yapflib/reformatter.py index 75482c50e..eeac49873 100644 --- a/yapf/yapflib/reformatter.py +++ b/yapf/yapflib/reformatter.py @@ -1,4 +1,4 @@ -# Copyright 2015-2016 Google Inc. All Rights Reserved. +# Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,109 +13,128 @@ # limitations under the License. """Decide what the format for the code should be. -The `unwrapped_line.UnwrappedLine`s are now ready to be formatted. -UnwrappedLines that can be merged together are. The best formatting is returned -as a string. +The `logical_line.LogicalLine`s are now ready to be formatted. LogicalLInes that +can be merged together are. The best formatting is returned as a string. Reformat(): the main function exported by this module. """ -from __future__ import unicode_literals import collections import heapq import re -from lib2to3 import pytree -from lib2to3.pgen2 import token +from yapf_third_party._ylib2to3 import pytree +from yapf_third_party._ylib2to3.pgen2 import token +from yapf.pytree import pytree_utils from yapf.yapflib import format_decision_state from yapf.yapflib import format_token from yapf.yapflib import line_joiner -from yapf.yapflib import pytree_utils from yapf.yapflib import style -from yapf.yapflib import verifier -def Reformat(uwlines, verify=True): - """Reformat the unwrapped lines. +def Reformat(llines, lines=None): + """Reformat the logical lines. Arguments: - uwlines: (list of unwrapped_line.UnwrappedLine) Lines we want to format. - verify: (bool) True if reformatted code should be verified for syntax. + llines: (list of logical_line.LogicalLine) Lines we want to format. + lines: (set of int) The lines which can be modified or None if there is no + line range restriction. Returns: A string representing the reformatted code. """ final_lines = [] - prev_uwline = None # The previous line. + prev_line = None # The previous line. indent_width = style.Get('INDENT_WIDTH') - for uwline in _SingleOrMergedLines(uwlines): - first_token = uwline.first - _FormatFirstToken(first_token, uwline.depth, prev_uwline, final_lines) + for lline in _SingleOrMergedLines(llines): + first_token = lline.first + _FormatFirstToken(first_token, lline.depth, prev_line, final_lines) - indent_amt = indent_width * uwline.depth - state = format_decision_state.FormatDecisionState(uwline, indent_amt) + indent_amt = indent_width * lline.depth + state = format_decision_state.FormatDecisionState(lline, indent_amt) + state.MoveStateToNextToken() - if not uwline.disable: - if uwline.first.is_comment: - uwline.first.node.value = uwline.first.node.value.rstrip() - elif uwline.last.is_comment: - uwline.last.node.value = uwline.last.node.value.rstrip() - if prev_uwline and prev_uwline.disable: + if not lline.disable: + if lline.first.is_comment: + lline.first.value = lline.first.value.rstrip() + elif lline.last.is_comment: + lline.last.value = lline.last.value.rstrip() + if prev_line and prev_line.disable: # Keep the vertical spacing between a disabled and enabled formatting # region. - _RetainVerticalSpacingBetweenTokens(uwline.first, prev_uwline.last) - if any(tok.is_comment for tok in uwline.tokens): - _RetainVerticalSpacingBeforeComments(uwline) - - if (_LineContainsI18n(uwline) or uwline.disable or - _LineHasContinuationMarkers(uwline)): - _RetainHorizontalSpacing(uwline) - _RetainVerticalSpacing(uwline, prev_uwline) + _RetainRequiredVerticalSpacingBetweenTokens(lline.first, prev_line.last, + lines) + if any(tok.is_comment for tok in lline.tokens): + _RetainVerticalSpacingBeforeComments(lline) + + if lline.disable or _LineHasContinuationMarkers(lline): + _RetainHorizontalSpacing(lline) + _RetainRequiredVerticalSpacing(lline, prev_line, lines) _EmitLineUnformatted(state) - elif (_CanPlaceOnSingleLine(uwline) and not any(tok.must_split - for tok in uwline.tokens)): - # The unwrapped line fits on one line. + + elif (_LineContainsPylintDisableLineTooLong(lline) or + _LineContainsI18n(lline)): + # Don't modify vertical spacing, but fix any horizontal spacing issues. + _RetainRequiredVerticalSpacing(lline, prev_line, lines) + _EmitLineUnformatted(state) + + elif _CanPlaceOnSingleLine(lline) and not any(tok.must_break_before + for tok in lline.tokens): + # The logical line fits on one line. while state.next_token: state.AddTokenToState(newline=False, dry_run=False) - else: - if not _AnalyzeSolutionSpace(state): - # Failsafe mode. If there isn't a solution to the line, then just emit - # it as is. - state = format_decision_state.FormatDecisionState(uwline, indent_amt) - _RetainHorizontalSpacing(uwline) - _RetainVerticalSpacing(uwline, prev_uwline) - _EmitLineUnformatted(state) - final_lines.append(uwline) - prev_uwline = uwline - return _FormatFinalLines(final_lines, verify) + elif not _AnalyzeSolutionSpace(state): + # Failsafe mode. If there isn't a solution to the line, then just emit + # it as is. + state = format_decision_state.FormatDecisionState(lline, indent_amt) + state.MoveStateToNextToken() + _RetainHorizontalSpacing(lline) + _RetainRequiredVerticalSpacing(lline, prev_line, None) + _EmitLineUnformatted(state) + + final_lines.append(lline) + prev_line = lline + if style.Get('ALIGN_ASSIGNMENT'): + _AlignAssignment(final_lines) -def _RetainHorizontalSpacing(uwline): + _AlignTrailingComments(final_lines) + return _FormatFinalLines(final_lines) + + +def _RetainHorizontalSpacing(line): """Retain all horizontal spacing between tokens.""" - for tok in uwline.tokens: - tok.RetainHorizontalSpacing(uwline.first.column, uwline.depth) + for tok in line.tokens: + tok.RetainHorizontalSpacing(line.first.column, line.depth) -def _RetainVerticalSpacing(cur_uwline, prev_uwline): +def _RetainRequiredVerticalSpacing(cur_line, prev_line, lines): + """Retain all vertical spacing between lines.""" prev_tok = None - if prev_uwline is not None: - prev_tok = prev_uwline.last - for cur_tok in cur_uwline.tokens: - _RetainVerticalSpacingBetweenTokens(cur_tok, prev_tok) + if prev_line is not None: + prev_tok = prev_line.last + + if cur_line.disable: + # After the first token we are acting on a single line. So if it is + # disabled we must not reformat. + lines = set() + + for cur_tok in cur_line.tokens: + _RetainRequiredVerticalSpacingBetweenTokens(cur_tok, prev_tok, lines) prev_tok = cur_tok -def _RetainVerticalSpacingBetweenTokens(cur_tok, prev_tok): - """Retain vertical spacing between two tokens.""" +def _RetainRequiredVerticalSpacingBetweenTokens(cur_tok, prev_tok, lines): + """Retain vertical spacing between two tokens if not in editable range.""" if prev_tok is None: return if prev_tok.is_string: prev_lineno = prev_tok.lineno + prev_tok.value.count('\n') - elif prev_tok.is_pseudo_paren: + elif prev_tok.is_pseudo: if not prev_tok.previous_token.is_multiline_string: prev_lineno = prev_tok.previous_token.lineno else: @@ -128,13 +147,27 @@ def _RetainVerticalSpacingBetweenTokens(cur_tok, prev_tok): else: cur_lineno = cur_tok.lineno - cur_tok.AdjustNewlinesBefore(cur_lineno - prev_lineno) + if not prev_tok.is_comment and prev_tok.value.endswith('\\'): + prev_lineno += prev_tok.value.count('\n') + + required_newlines = cur_lineno - prev_lineno + if cur_tok.is_comment and not prev_tok.is_comment: + # Don't adjust between a comment and non-comment. + pass + elif lines and lines.intersection(range(prev_lineno, cur_lineno + 1)): + desired_newlines = cur_tok.whitespace_prefix.count('\n') + whitespace_lines = range(prev_lineno + 1, cur_lineno) + deletable_lines = len(lines.intersection(whitespace_lines)) + required_newlines = max(required_newlines - deletable_lines, + desired_newlines) + + cur_tok.AdjustNewlinesBefore(required_newlines) -def _RetainVerticalSpacingBeforeComments(uwline): +def _RetainVerticalSpacingBeforeComments(line): """Retain vertical spacing before comments.""" prev_token = None - for tok in uwline.tokens: + for tok in line.tokens: if tok.is_comment and prev_token: if tok.lineno - tok.value.count('\n') - prev_token.lineno > 1: tok.AdjustNewlinesBefore(ONE_BLANK_LINE) @@ -154,91 +187,377 @@ def _EmitLineUnformatted(state): state: (format_decision_state.FormatDecisionState) The format decision state. """ - prev_lineno = None while state.next_token: previous_token = state.next_token.previous_token previous_lineno = previous_token.lineno - if previous_token.is_multiline_string: + if previous_token.is_multiline_string or previous_token.is_string: previous_lineno += previous_token.value.count('\n') if previous_token.is_continuation: newline = False else: - newline = (prev_lineno is not None and - state.next_token.lineno > previous_lineno) + newline = state.next_token.lineno > previous_lineno - prev_lineno = state.next_token.lineno state.AddTokenToState(newline=newline, dry_run=False) -def _LineContainsI18n(uwline): +def _LineContainsI18n(line): """Return true if there are i18n comments or function calls in the line. I18n comments and pseudo-function calls are closely related. They cannot be moved apart without breaking i18n. Arguments: - uwline: (unwrapped_line.UnwrappedLine) The line currently being formatted. + line: (logical_line.LogicalLine) The line currently being formatted. Returns: True if the line contains i18n comments or function calls. False otherwise. """ if style.Get('I18N_COMMENT'): - for tok in uwline.tokens: + for tok in line.tokens: if tok.is_comment and re.match(style.Get('I18N_COMMENT'), tok.value): # Contains an i18n comment. return True if style.Get('I18N_FUNCTION_CALL'): - length = len(uwline.tokens) - index = 0 - while index < length - 1: - if (uwline.tokens[index + 1].value == '(' and - uwline.tokens[index].value in style.Get('I18N_FUNCTION_CALL')): + length = len(line.tokens) + for index in range(length - 1): + if (line.tokens[index + 1].value == '(' and + line.tokens[index].value in style.Get('I18N_FUNCTION_CALL')): return True - index += 1 - return False -def _LineHasContinuationMarkers(uwline): +def _LineContainsPylintDisableLineTooLong(line): + """Return true if there is a "pylint: disable=line-too-long" comment.""" + return re.search(r'\bpylint:\s+disable=line-too-long\b', line.last.value) + + +def _LineHasContinuationMarkers(line): """Return true if the line has continuation markers in it.""" - return any(tok.is_continuation for tok in uwline.tokens) + return any(tok.is_continuation for tok in line.tokens) -def _CanPlaceOnSingleLine(uwline): - """Determine if the unwrapped line can go on a single line. +def _CanPlaceOnSingleLine(line): + """Determine if the logical line can go on a single line. Arguments: - uwline: (unwrapped_line.UnwrappedLine) The line currently being formatted. + line: (logical_line.LogicalLine) The line currently being formatted. Returns: True if the line can or should be added to a single line. False otherwise. """ - indent_amt = style.Get('INDENT_WIDTH') * uwline.depth - return (uwline.last.total_length + indent_amt <= style.Get('COLUMN_LIMIT') and - not any(tok.is_comment for tok in uwline.tokens[:-1])) + token_types = [x.type for x in line.tokens] + if (style.Get('SPLIT_ARGUMENTS_WHEN_COMMA_TERMINATED') and + any(token_types[token_index - 1] == token.COMMA + for token_index, token_type in enumerate(token_types[1:], start=1) + if token_type == token.RPAR)): + return False + if (style.Get('FORCE_MULTILINE_DICT') and token.LBRACE in token_types): + return False + indent_amt = style.Get('INDENT_WIDTH') * line.depth + last = line.last + last_index = -1 + if (last.is_pylint_comment or last.is_pytype_comment or + last.is_copybara_comment): + last = last.previous_token + last_index = -2 + if last is None: + return True + return (last.total_length + indent_amt <= style.Get('COLUMN_LIMIT') and + not any(tok.is_comment for tok in line.tokens[:last_index])) + + +def _AlignGeneric(final_lines, func_is_tok_to_align_current_line, + func_additional_checks_break_align_block, + func_calculate_pre_token_line_lengths, + func_calculate_column_to_align_on, + func_is_tok_to_align_following_lines, + func_get_val_after_align): + """Implements generic alignement logic which needs to be complemented for + needed purpose by passing suitable functions as parameters. + """ + final_lines_index = 0 + while final_lines_index < len(final_lines): + line = final_lines[final_lines_index] + assert line.tokens + + processed_content = False + + for tok in line.tokens: + + if func_is_tok_to_align_current_line(tok): + all_pt_line_lengths = [] # All pre-tok line lengths + max_line_length = 0 + stop_next_it = False + + while not stop_next_it: + # EOF + if final_lines_index + len(all_pt_line_lengths) == len(final_lines): + break + + this_line = final_lines[final_lines_index + len(all_pt_line_lengths)] + + # Blank line - note that content is preformatted so we don't need to + # worry about spaces/tabs; a blank line will always be '\n\n'. + assert this_line.tokens + if (all_pt_line_lengths and + this_line.tokens[0].formatted_whitespace_prefix.startswith('\n\n') + ): + break + + if this_line.disable: + all_pt_line_lengths.append([]) + continue + + # Check for additional specific conditions for breaking out of loop. + stop_now, stop_next_it = func_additional_checks_break_align_block( + tok, final_lines, final_lines_index, all_pt_line_lengths) + if stop_now: + break + + # Calculate the length of each line in this logical line. + pt_line_lengths, max_line_length, tmp = func_calculate_pre_token_line_lengths( + this_line, max_line_length) + stop_next_it = stop_next_it or tmp + + if pt_line_lengths: + max_line_length = max(max_line_length, max(pt_line_lengths)) + + all_pt_line_lengths.append(pt_line_lengths) + + # Calculate the aligned column value + max_line_length += 2 + aligned_col = func_calculate_column_to_align_on(tok, max_line_length) + assert isinstance(aligned_col, int) + + # Update the token values based on the aligned values + for all_pt_line_lengths_index, pt_line_lengths in enumerate( + all_pt_line_lengths): + if not pt_line_lengths: + continue + + this_line = final_lines[final_lines_index + all_pt_line_lengths_index] + + pt_line_length_index = 0 + for line_tok in this_line.tokens: + if func_is_tok_to_align_following_lines(line_tok): + assert pt_line_length_index < len(pt_line_lengths) + assert pt_line_lengths[pt_line_length_index] < aligned_col + line_tok.value = func_get_val_after_align( + line_tok, aligned_col, pt_line_lengths[pt_line_length_index]) + pt_line_length_index += 1 + + assert pt_line_length_index == len(pt_line_lengths) + + final_lines_index += len(all_pt_line_lengths) + + processed_content = True + break + + if not processed_content: + final_lines_index += 1 + + +def _AlignTrailingComments(final_lines): + """Align trailing comments to the same column.""" + + def _IsTokToAlignCurrentLine(tok): + # All trailing comments and comments that appear on a line by themselves + # in this block should be indented at the same level. The block is + # terminated by an empty line or EOF. Enumerate through each line in + # the block and calculate the max line length. Once complete, use the + # first col value greater than that value and create the necessary for + # each line accordingly. + return tok.is_comment and isinstance(tok.spaces_required_before, + list) and tok.value.startswith('#') + + def _IsTokToAlignFollowingLines(tok): + return tok.is_comment + + def _AdditionalChecksBreakAlignBlock(tok, final_lines, final_lines_index, + all_pt_line_lengths): + return False, False + + def _CalculatePreTokenLineLengths(this_line, max_line_length): + line_content = '' + pt_line_lengths = [] + stop_align_block = False + + for line_tok in this_line.tokens: + whitespace_prefix = line_tok.formatted_whitespace_prefix + + newline_index = whitespace_prefix.rfind('\n') + if newline_index != -1: + max_line_length = max(max_line_length, len(line_content)) + line_content = '' + + whitespace_prefix = whitespace_prefix[newline_index + 1:] + + if line_tok.is_comment: + pt_line_lengths.append(len(line_content)) + else: + line_content += '{}{}'.format(whitespace_prefix, line_tok.value) + return pt_line_lengths, max_line_length, stop_align_block -def _FormatFinalLines(final_lines, verify): + def _CalculateColumnToAlignOn(tok, max_line_length): + aligned_col = None + for potential_col in tok.spaces_required_before: + if potential_col > max_line_length: + aligned_col = potential_col + break + + if aligned_col is None: + aligned_col = max_line_length + + return aligned_col + + def _GetValueAfterAlign(line_tok, aligned_col, pt_line_length): + # Note that there may be newlines embedded in the comments, so we need to + # apply a whitespace prefix to each line. + whitespace = ' ' * (aligned_col - pt_line_length - 1) + + line_content = [] + + for comment_line_index, comment_line in enumerate( + line_tok.value.split('\n')): + line_content.append('{}{}'.format(whitespace, comment_line.strip())) + + if comment_line_index == 0: + whitespace = ' ' * (aligned_col - 1) + + line_content = '\n'.join(line_content) + + # Account for initial whitespace already slated for beginning of the line. + existing_whitespace_prefix = line_tok.formatted_whitespace_prefix.lstrip( + '\n') + + if line_content.startswith(existing_whitespace_prefix): + line_content = line_content[len(existing_whitespace_prefix):] + + return line_content + + return _AlignGeneric(final_lines, _IsTokToAlignCurrentLine, + _AdditionalChecksBreakAlignBlock, + _CalculatePreTokenLineLengths, _CalculateColumnToAlignOn, + _IsTokToAlignFollowingLines, _GetValueAfterAlign) + + +def _AlignAssignment(final_lines): + """Align assignment operators and augmented assignment operators to the same column""" + + def _IsTokToAlign(tok): + return tok.is_assign or tok.is_augassign + + def _AdditionalChecksBreakAlignBlock(tok, final_lines, final_lines_index, + all_pt_line_lengths): + stop_now = False + stop_next_it = False + + this_line_index = final_lines_index + len(all_pt_line_lengths) + this_line = final_lines[this_line_index] + + if this_line_index < len(final_lines) - 1: + next_line = final_lines[this_line_index + 1] + assert next_line.tokens + + if this_line.depth != next_line.depth: + stop_next_it = True + + # If there is a standalone comment or keyword statement line or other lines + # without assignment in between, break. + if (all_pt_line_lengths and + not any(tok.is_assign or tok.is_augassign for tok in this_line.tokens)): + if this_line.tokens[0].is_comment: + if style.Get('ALIGN_ASSIGNMENT_RESTART_AFTER_COMMENTS'): + stop_now = True + else: + stop_now = True + return stop_now, stop_next_it + + def _CalculatePreTokenLineLengths(this_line, max_line_length): + pt_line_length = [] + variables_content = '' + contain_object = False + line_tokens = this_line.tokens + # Only one assignment expression in on each line + for index in range(len(line_tokens)): + line_tok = line_tokens[index] + + prefix = line_tok.formatted_whitespace_prefix + newline_index = prefix.rfind('\n') + if newline_index != -1: + variables_content = '' + prefix = prefix[newline_index + 1:] + + if line_tok.is_assign or line_tok.is_augassign: + next_toks = [line_tokens[i] for i in range(index + 1, len(line_tokens))] + # if there is object(list/tuple/dict) with newline entries, break, + # update the alignment so far and start to calulate new alignment + for tok in next_toks: + if tok.value in ['(', '[', '{'] and tok.next_token: + if (tok.next_token.formatted_whitespace_prefix.startswith('\n') or + (tok.next_token.is_comment and tok.next_token.next_token + .formatted_whitespace_prefix.startswith('\n'))): + pt_line_length.append(len(variables_content)) + contain_object = True + break + if not contain_object: + if line_tok.is_assign: + pt_line_length.append(len(variables_content)) + # if augassign, add the extra augmented part to the max length caculation + elif line_tok.is_augassign: + pt_line_length.append( + len(variables_content) + len(line_tok.value) - 1) + # don't add the tokens after the assignment operator + break + else: + variables_content += '{}{}'.format(prefix, line_tok.value) + return pt_line_length, max_line_length, contain_object + + def _CalculateColumnToAlignOn(tok, max_line_length): + return max_line_length + + def _GetValueAfterAlign(line_tok, aligned_col, pt_line_length): + whitespace = ' ' * (aligned_col - pt_line_length - 1) + + assign_content = '{}{}'.format(whitespace, line_tok.value.strip()) + + existing_whitespace_prefix = line_tok.formatted_whitespace_prefix.lstrip( + '\n') + + # In case the existing spaces are larger than padded spaces + if (len(whitespace) == 1 or len(whitespace) > 1 and + len(existing_whitespace_prefix) > len(whitespace)): + line_tok.whitespace_prefix = '' + elif assign_content.startswith(existing_whitespace_prefix): + assign_content = assign_content[len(existing_whitespace_prefix):] + return assign_content + + return _AlignGeneric(final_lines, _IsTokToAlign, + _AdditionalChecksBreakAlignBlock, + _CalculatePreTokenLineLengths, _CalculateColumnToAlignOn, + _IsTokToAlign, _GetValueAfterAlign) + + +def _FormatFinalLines(final_lines): + """Compose the final output from the finalized lines.""" formatted_code = [] for line in final_lines: formatted_line = [] for tok in line.tokens: - if not tok.is_pseudo_paren: - formatted_line.append(tok.whitespace_prefix) + if not tok.is_pseudo: + formatted_line.append(tok.formatted_whitespace_prefix) formatted_line.append(tok.value) - else: - if (not tok.next_token.whitespace_prefix.startswith('\n') and + elif (not tok.next_token.whitespace_prefix.startswith('\n') and not tok.next_token.whitespace_prefix.startswith(' ')): - if (tok.previous_token.value == ':' or - tok.next_token.value not in ',}])'): - formatted_line.append(' ') + if (tok.previous_token.value == ':' or + tok.next_token.value not in ',}])'): + formatted_line.append(' ') formatted_code.append(''.join(formatted_line)) - if verify: - verifier.VerifyCode(formatted_code[-1]) return ''.join(formatted_code) + '\n' @@ -262,8 +581,9 @@ def __init__(self, state, newline, previous): self.previous = previous def __repr__(self): # pragma: no cover - return 'StateNode(state=[\n{0}\n], newline={1})'.format(self.state, - self.newline) + return 'StateNode(state=[\n{0}\n], newline={1})'.format( + self.state, self.newline) + # A tuple of (penalty, count) that is used to prioritize the BFS. In case of # equal penalties, we prefer states that were inserted first. During state @@ -273,8 +593,8 @@ def __repr__(self): # pragma: no cover # An item in the prioritized BFS search queue. The 'StateNode's 'state' has # the given '_OrderedPenalty'. -_QueueItem = collections.namedtuple('QueueItem', ['ordered_penalty', - 'state_node']) +_QueueItem = collections.namedtuple('QueueItem', + ['ordered_penalty', 'state_node']) def _AnalyzeSolutionSpace(initial_state): @@ -301,7 +621,6 @@ def _AnalyzeSolutionSpace(initial_state): heapq.heappush(p_queue, _QueueItem(_OrderedPenalty(0, count), node)) count += 1 - prev_penalty = 0 while p_queue: item = p_queue[0] penalty = item.ordered_penalty.penalty @@ -313,11 +632,13 @@ def _AnalyzeSolutionSpace(initial_state): if count > 10000: node.state.ignore_stack_for_comparison = True - if node.state in seen: - continue - - prev_penalty = penalty + # Unconditionally add the state and check if it was present to avoid having + # to hash it twice in the common case (state hashing is expensive). + before_seen_count = len(seen) seen.add(node.state) + # If seen didn't change size, the state was already present. + if before_seen_count == len(seen): + continue # FIXME(morbo): Add a 'decision' element? @@ -349,10 +670,10 @@ def _AddNextStateToQueue(penalty, previous_node, newline, count, p_queue): Returns: The updated number of elements in the queue. """ - if newline and not previous_node.state.CanSplit(): + must_split = previous_node.state.MustSplit() + if newline and not previous_node.state.CanSplit(must_split): # Don't add a newline if the token cannot be split. return count - must_split = previous_node.state.MustSplit() if not newline and must_split: # Don't add a token we must split but where we aren't splitting. return count @@ -383,24 +704,38 @@ def _ReconstructPath(initial_state, current): initial_state.AddTokenToState(newline=node.newline, dry_run=False) -def _FormatFirstToken(first_token, indent_depth, prev_uwline, final_lines): - """Format the first token in the unwrapped line. +NESTED_DEPTH = [] + + +def _FormatFirstToken(first_token, indent_depth, prev_line, final_lines): + """Format the first token in the logical line. - Add a newline and the required indent before the first token of the unwrapped + Add a newline and the required indent before the first token of the logical line. Arguments: - first_token: (format_token.FormatToken) The first token in the unwrapped - line. + first_token: (format_token.FormatToken) The first token in the logical line. indent_depth: (int) The line's indentation depth. - prev_uwline: (list of unwrapped_line.UnwrappedLine) The unwrapped line - previous to this line. - final_lines: (list of unwrapped_line.UnwrappedLine) The unwrapped lines - that have already been processed. + prev_line: (list of logical_line.LogicalLine) The logical line previous to + this line. + final_lines: (list of logical_line.LogicalLine) The logical lines that have + already been processed. """ + global NESTED_DEPTH + while NESTED_DEPTH and NESTED_DEPTH[-1] > indent_depth: + NESTED_DEPTH.pop() + + first_nested = False + if _IsClassOrDef(first_token): + if not NESTED_DEPTH: + NESTED_DEPTH = [indent_depth] + elif NESTED_DEPTH[-1] < indent_depth: + first_nested = True + NESTED_DEPTH.append(indent_depth) + first_token.AddWhitespacePrefix( - _CalculateNumberOfNewlines(first_token, indent_depth, prev_uwline, - final_lines), + _CalculateNumberOfNewlines(first_token, indent_depth, prev_line, + final_lines, first_nested), indent_level=indent_depth) @@ -409,56 +744,83 @@ def _FormatFirstToken(first_token, indent_depth, prev_uwline, final_lines): TWO_BLANK_LINES = 3 -def _CalculateNumberOfNewlines(first_token, indent_depth, prev_uwline, - final_lines): +def _IsClassOrDef(tok): + if tok.value in {'class', 'def', '@'}: + return True + return (tok.next_token and tok.value == 'async' and + tok.next_token.value == 'def') + + +def _CalculateNumberOfNewlines(first_token, indent_depth, prev_line, + final_lines, first_nested): """Calculate the number of newlines we need to add. Arguments: - first_token: (format_token.FormatToken) The first token in the unwrapped + first_token: (format_token.FormatToken) The first token in the logical line. indent_depth: (int) The line's indentation depth. - prev_uwline: (list of unwrapped_line.UnwrappedLine) The unwrapped line - previous to this line. - final_lines: (list of unwrapped_line.UnwrappedLine) The unwrapped lines - that have already been processed. + prev_line: (list of logical_line.LogicalLine) The logical line previous to + this line. + final_lines: (list of logical_line.LogicalLine) The logical lines that have + already been processed. + first_nested: (boolean) Whether this is the first nested class or function. Returns: The number of newlines needed before the first token. """ # TODO(morbo): Special handling for imports. # TODO(morbo): Create a knob that can tune these. - if prev_uwline is None: + if prev_line is None: # The first line in the file. Don't add blank lines. # FIXME(morbo): Is this correct? if first_token.newlines is not None: - pytree_utils.SetNodeAnnotation(first_token.node, - pytree_utils.Annotation.NEWLINES, None) + first_token.newlines = None return 0 if first_token.is_docstring: + if (prev_line.first.value == 'class' and + style.Get('BLANK_LINE_BEFORE_CLASS_DOCSTRING')): + # Enforce a blank line before a class's docstring. + return ONE_BLANK_LINE + elif (prev_line.first.value.startswith('#') and + style.Get('BLANK_LINE_BEFORE_MODULE_DOCSTRING')): + # Enforce a blank line before a module's docstring. + return ONE_BLANK_LINE # The docstring shouldn't have a newline before it. - # TODO(morbo): Add a knob to adjust this. return NO_BLANK_LINES - prev_last_token = prev_uwline.last + if first_token.is_name and not indent_depth: + if prev_line.first.value in {'from', 'import'}: + # Support custom number of blank lines between top-level imports and + # variable definitions. + return 1 + style.Get( + 'BLANK_LINES_BETWEEN_TOP_LEVEL_IMPORTS_AND_VARIABLES') + + prev_last_token = prev_line.last if prev_last_token.is_docstring: if (not indent_depth and first_token.value in {'class', 'def', 'async'}): - # Separate a class or function from the module-level docstring with two - # blank lines. - return TWO_BLANK_LINES + # Separate a class or function from the module-level docstring with + # appropriate number of blank lines. + return 1 + style.Get('BLANK_LINES_AROUND_TOP_LEVEL_DEFINITION') + if (first_nested and + not style.Get('BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF') and + _IsClassOrDef(first_token)): + first_token.newlines = None + return NO_BLANK_LINES if _NoBlankLinesBeforeCurrentToken(prev_last_token.value, first_token, prev_last_token): return NO_BLANK_LINES else: return ONE_BLANK_LINE - if first_token.value in {'class', 'def', '@'}: + if _IsClassOrDef(first_token): # TODO(morbo): This can go once the blank line calculator is more # sophisticated. if not indent_depth: # This is a top-level class or function. is_inline_comment = prev_last_token.whitespace_prefix.count('\n') == 0 - if prev_last_token.is_comment and not is_inline_comment: + if (not prev_line.disable and prev_last_token.is_comment and + not is_inline_comment): # This token follows a non-inline comment. if _NoBlankLinesBeforeCurrentToken(prev_last_token.value, first_token, prev_last_token): @@ -472,16 +834,15 @@ def _CalculateNumberOfNewlines(first_token, indent_depth, prev_uwline, if final_lines[index - 1].first.value == '@': final_lines[index].first.AdjustNewlinesBefore(NO_BLANK_LINES) else: - prev_last_token.AdjustNewlinesBefore(TWO_BLANK_LINES) + prev_last_token.AdjustNewlinesBefore( + 1 + style.Get('BLANK_LINES_AROUND_TOP_LEVEL_DEFINITION')) if first_token.newlines is not None: - pytree_utils.SetNodeAnnotation(first_token.node, - pytree_utils.Annotation.NEWLINES, - None) + first_token.newlines = None return NO_BLANK_LINES - elif prev_uwline.first.value in {'class', 'def'}: - if not style.Get('BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF'): - pytree_utils.SetNodeAnnotation(first_token.node, - pytree_utils.Annotation.NEWLINES, None) + elif _IsClassOrDef(prev_line.first): + if first_nested and not style.Get( + 'BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF'): + first_token.newlines = None return NO_BLANK_LINES # Calculate how many newlines were between the original lines. We want to @@ -501,11 +862,11 @@ def _CalculateNumberOfNewlines(first_token, indent_depth, prev_uwline, return NO_BLANK_LINES -def _SingleOrMergedLines(uwlines): +def _SingleOrMergedLines(lines): """Generate the lines we want to format. Arguments: - uwlines: (list of unwrapped_line.UnwrappedLine) Lines we want to format. + lines: (list of logical_line.LogicalLine) Lines we want to format. Yields: Either a single line, if the current line cannot be merged with the @@ -513,38 +874,38 @@ def _SingleOrMergedLines(uwlines): """ index = 0 last_was_merged = False - while index < len(uwlines): - if uwlines[index].disable: - uwline = uwlines[index] + while index < len(lines): + if lines[index].disable: + line = lines[index] index += 1 - while index < len(uwlines): - column = uwline.last.column + 2 - if uwlines[index].lineno != uwline.lineno: + while index < len(lines): + column = line.last.column + 2 + if lines[index].lineno != line.lineno: break - if uwline.last.value != ':': + if line.last.value != ':': leaf = pytree.Leaf( - type=token.SEMI, value=';', context=('', (uwline.lineno, column))) - uwline.AppendToken(format_token.FormatToken(leaf)) - for tok in uwlines[index].tokens: - uwline.AppendToken(tok) + type=token.SEMI, value=';', context=('', (line.lineno, column))) + line.AppendToken( + format_token.FormatToken(leaf, pytree_utils.NodeName(leaf))) + for tok in lines[index].tokens: + line.AppendToken(tok) index += 1 - yield uwline - elif line_joiner.CanMergeMultipleLines(uwlines[index:], last_was_merged): + yield line + elif line_joiner.CanMergeMultipleLines(lines[index:], last_was_merged): # TODO(morbo): This splice is potentially very slow. Come up with a more # performance-friendly way of determining if two lines can be merged. - next_uwline = uwlines[index + 1] - for tok in next_uwline.tokens: - uwlines[index].AppendToken(tok) - if (len(next_uwline.tokens) == 1 and - next_uwline.first.is_multiline_string): + next_line = lines[index + 1] + for tok in next_line.tokens: + lines[index].AppendToken(tok) + if (len(next_line.tokens) == 1 and next_line.first.is_multiline_string): # This may be a multiline shebang. In that case, we want to retain the # formatting. Otherwise, it could mess up the shell script's syntax. - uwlines[index].disable = True - yield uwlines[index] + lines[index].disable = True + yield lines[index] index += 2 last_was_merged = True else: - yield uwlines[index] + yield lines[index] index += 1 last_was_merged = False @@ -561,9 +922,8 @@ def _NoBlankLinesBeforeCurrentToken(text, cur_token, prev_token): Arguments: text: (unicode) The text of the docstring or comment before the current token. - cur_token: (format_token.FormatToken) The current token in the unwrapped - line. - prev_token: (format_token.FormatToken) The previous token in the unwrapped + cur_token: (format_token.FormatToken) The current token in the logical line. + prev_token: (format_token.FormatToken) The previous token in the logical line. Returns: diff --git a/yapf/yapflib/split_penalty.py b/yapf/yapflib/split_penalty.py index 50796decf..79b68edcd 100644 --- a/yapf/yapflib/split_penalty.py +++ b/yapf/yapflib/split_penalty.py @@ -1,4 +1,4 @@ -# Copyright 2015-2016 Google Inc. All Rights Reserved. +# Copyright 2022 Bill Wendling, All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,375 +11,29 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Computation of split penalties before/between tokens.""" -from lib2to3 import pytree - -from yapf.yapflib import format_token -from yapf.yapflib import py3compat -from yapf.yapflib import pytree_utils -from yapf.yapflib import pytree_visitor from yapf.yapflib import style -# TODO(morbo): Document the annotations in a centralized place. E.g., the -# README file. -UNBREAKABLE = 1000 * 1000 -DOTTED_NAME = 2500 -STRONGLY_CONNECTED = 2000 -CONTIGUOUS_LIST = 500 - -NOT_TEST = 242 -COMPARISON_EXPRESSION = 842 -ARITHMETIC_EXPRESSION = 942 - - -def ComputeSplitPenalties(tree): - """Compute split penalties on tokens in the given parse tree. - - Arguments: - tree: the top-level pytree node to annotate with penalties. - """ - _TreePenaltyAssigner().Visit(tree) - - -class _TreePenaltyAssigner(pytree_visitor.PyTreeVisitor): - """Assigns split penalties to tokens, based on parse tree structure. - - Split penalties are attached as annotations to tokens. - """ - - def Visit_import_as_names(self, node): # pyline: disable=invalid-name - # import_as_names ::= import_as_name (',' import_as_name)* [','] - self.DefaultNodeVisit(node) - prev_child = None - for child in node.children: - if (prev_child and isinstance(prev_child, pytree.Leaf) and - prev_child.value == ','): - pytree_utils.SetNodeAnnotation(child, - pytree_utils.Annotation.SPLIT_PENALTY, - style.Get('SPLIT_PENALTY_IMPORT_NAMES')) - prev_child = child - - def Visit_classdef(self, node): # pylint: disable=invalid-name - # classdef ::= 'class' NAME ['(' [arglist] ')'] ':' suite - # - # NAME - self._SetUnbreakable(node.children[1]) - if len(node.children) > 4: - # opening '(' - self._SetUnbreakable(node.children[2]) - # ':' - self._SetUnbreakable(node.children[-2]) - self.DefaultNodeVisit(node) - - def Visit_funcdef(self, node): # pylint: disable=invalid-name - # funcdef ::= 'def' NAME parameters ['->' test] ':' suite - # - # Can't break before the function name and before the colon. The parameters - # are handled by child iteration. - colon_idx = 1 - while pytree_utils.NodeName(node.children[colon_idx]) == 'simple_stmt': - colon_idx += 1 - self._SetUnbreakable(node.children[colon_idx]) - while colon_idx < len(node.children): - if (isinstance(node.children[colon_idx], pytree.Leaf) and - node.children[colon_idx].value == ':'): - break - colon_idx += 1 - self._SetUnbreakable(node.children[colon_idx]) - self.DefaultNodeVisit(node) - - def Visit_lambdef(self, node): # pylint: disable=invalid-name - # lambdef ::= 'lambda' [varargslist] ':' test - # Loop over the lambda up to and including the colon. - if style.Get('ALLOW_MULTILINE_LAMBDAS'): - self._SetStronglyConnected(node) - else: - self._SetUnbreakableOnChildren(node) - - def Visit_parameters(self, node): # pylint: disable=invalid-name - # parameters ::= '(' [typedargslist] ')' - self.DefaultNodeVisit(node) - - # Can't break before the opening paren of a parameter list. - self._SetUnbreakable(node.children[0]) - if not style.Get('DEDENT_CLOSING_BRACKETS'): - self._SetStronglyConnected(node.children[-1]) - - def Visit_argument(self, node): # pylint: disable=invalid-name - # argument ::= test [comp_for] | test '=' test # Really [keyword '='] test - self.DefaultNodeVisit(node) - - index = 0 - while index < len(node.children) - 1: - next_child = node.children[index + 1] - if isinstance(next_child, pytree.Leaf) and next_child.value == '=': - self._SetUnbreakable(_FirstChildNode(node.children[index + 1])) - self._SetUnbreakable(_FirstChildNode(node.children[index + 2])) - index += 1 - - def Visit_dotted_name(self, node): # pylint: disable=invalid-name - # dotted_name ::= NAME ('.' NAME)* - self._SetUnbreakableOnChildren(node) - - def Visit_dictsetmaker(self, node): # pylint: disable=invalid-name - # dictsetmaker ::= ( (test ':' test - # (comp_for | (',' test ':' test)* [','])) | - # (test (comp_for | (',' test)* [','])) ) - for child in node.children: - self.Visit(child) - if pytree_utils.NodeName(child) == 'COLON': - # This is a key to a dictionary. We don't want to split the key if at - # all possible. - self._SetStronglyConnected(child) - - def Visit_trailer(self, node): # pylint: disable=invalid-name - # trailer ::= '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME - self.DefaultNodeVisit(node) - if node.children[0].value == '.': - self._SetUnbreakableOnChildren(node) - pytree_utils.SetNodeAnnotation(node.children[1], - pytree_utils.Annotation.SPLIT_PENALTY, - DOTTED_NAME) - elif len(node.children) == 2: - # Don't split an empty argument list if at all possible. - self._SetStronglyConnected(node.children[1]) - elif len(node.children) == 3: - if pytree_utils.NodeName(node.children[1]) not in { - 'arglist', 'argument', 'term', 'or_test', 'and_test', 'comparison', - 'atom' - }: - # Don't split an argument list with one element if at all possible. - self._SetStronglyConnected(node.children[1], node.children[2]) - if pytree_utils.NodeName(node.children[-1]) == 'RSQB': - # Don't split the ending bracket of a subscript list. - self._SetStronglyConnected(node.children[-1]) - - def Visit_power(self, node): # pylint: disable=invalid-name,missing-docstring - # power ::= atom trailer* ['**' factor] - self.DefaultNodeVisit(node) - - # When atom is followed by a trailer, we can not break between them. - # E.g. arr[idx] - no break allowed between 'arr' and '['. - if (len(node.children) > 1 and - pytree_utils.NodeName(node.children[1]) == 'trailer'): - # children[1] itself is a whole trailer: we don't want to - # mark all of it as unbreakable, only its first token: (, [ or . - self._SetUnbreakable(node.children[1].children[0]) - - # A special case when there are more trailers in the sequence. Given: - # atom tr1 tr2 - # The last token of tr1 and the first token of tr2 comprise an unbreakable - # region. For example: foo.bar.baz(1) - # We can't put breaks between either of the '.', '(', or '[' and the names - # *preceding* them. - prev_trailer_idx = 1 - while prev_trailer_idx < len(node.children) - 1: - cur_trailer_idx = prev_trailer_idx + 1 - cur_trailer = node.children[cur_trailer_idx] - if pytree_utils.NodeName(cur_trailer) == 'trailer': - # Now we know we have two trailers one after the other - prev_trailer = node.children[prev_trailer_idx] - if prev_trailer.children[-1].value != ')': - # Set the previous node unbreakable if it's not a function call: - # atom tr1() tr2 - # It may be necessary (though undesirable) to split up a previous - # function call's parentheses to the next line. - self._SetStronglyConnected(prev_trailer.children[-1]) - self._SetStronglyConnected(cur_trailer.children[0]) - prev_trailer_idx = cur_trailer_idx - else: - break - - # We don't want to split before the last ')' of a function call. This also - # takes care of the special case of: - # atom tr1 tr2 ... trn - # where the 'tr#' are trailers that may end in a ')'. - for trailer in node.children[1:]: - if pytree_utils.NodeName(trailer) != 'trailer': - break - if trailer.children[0].value in '([': - if len(trailer.children) > 2: - subtypes = pytree_utils.GetNodeAnnotation( - trailer.children[0], pytree_utils.Annotation.SUBTYPE) - if subtypes and format_token.Subtype.SUBSCRIPT_BRACKET in subtypes: - self._SetStronglyConnected(_FirstChildNode(trailer.children[1])) - - last_child_node = _LastChildNode(trailer) - if last_child_node.value.strip().startswith('#'): - last_child_node = last_child_node.prev_sibling - if not style.Get('DEDENT_CLOSING_BRACKETS'): - self._SetUnbreakable(last_child_node) - - if _FirstChildNode(trailer).lineno == last_child_node.lineno: - # If the trailer was originally on one line, then try to keep it - # like that. - self._SetExpressionPenalty(trailer, CONTIGUOUS_LIST) - else: - # If the trailer's children are '()', then make it a strongly - # connected region. It's sometimes necessary, though undesirable, to - # split the two. - self._SetStronglyConnected(trailer.children[-1]) - - # If the original source has a "builder" style calls, then we should allow - # the reformatter to retain that. - _AllowBuilderStyleCalls(node) - - def Visit_subscript(self, node): # pylint: disable=invalid-name - # subscript ::= test | [test] ':' [test] [sliceop] - self._SetStronglyConnected(*node.children) - self.DefaultNodeVisit(node) - - def Visit_comp_for(self, node): # pylint: disable=invalid-name - # comp_for ::= 'for' exprlist 'in' testlist_safe [comp_iter] - pytree_utils.SetNodeAnnotation( - _FirstChildNode(node), pytree_utils.Annotation.SPLIT_PENALTY, 0) - self._SetStronglyConnected(*node.children[1:]) - self.DefaultNodeVisit(node) - - def Visit_comp_if(self, node): # pylint: disable=invalid-name - # comp_if ::= 'if' old_test [comp_iter] - pytree_utils.SetNodeAnnotation(node.children[0], - pytree_utils.Annotation.SPLIT_PENALTY, - style.Get('SPLIT_PENALTY_BEFORE_IF_EXPR')) - self._SetStronglyConnected(*node.children[1:]) - self.DefaultNodeVisit(node) - - def Visit_not_test(self, node): # pylint: disable=invalid-name - # not_test ::= 'not' not_test | comparison - self.DefaultNodeVisit(node) - self._SetExpressionPenalty(node, NOT_TEST) - - def Visit_comparison(self, node): # pylint: disable=invalid-name - # comparison ::= expr (comp_op expr)* - self.DefaultNodeVisit(node) - self._SetExpressionPenalty(node, COMPARISON_EXPRESSION) - - def Visit_arith_expr(self, node): # pylint: disable=invalid-name - # arith_expr ::= term (('+'|'-') term)* - self.DefaultNodeVisit(node) - self._SetExpressionPenalty(node, ARITHMETIC_EXPRESSION) - - def Visit_atom(self, node): # pylint: disable=invalid-name - # atom ::= ('(' [yield_expr|testlist_gexp] ')' - # '[' [listmaker] ']' | - # '{' [dictsetmaker] '}') - self.DefaultNodeVisit(node) - if node.children[0].value == '(': - if node.children[0].lineno == node.children[-1].lineno: - self._SetExpressionPenalty(node, CONTIGUOUS_LIST) - if node.children[-1].value == ')': - if pytree_utils.NodeName(node.parent) == 'if_stmt': - pytree_utils.SetNodeAnnotation(node.children[-1], - pytree_utils.Annotation.SPLIT_PENALTY, - UNBREAKABLE) - else: - pytree_utils.SetNodeAnnotation(node.children[-1], - pytree_utils.Annotation.SPLIT_PENALTY, - COMPARISON_EXPRESSION) - elif node.children[0].value in '[{': - # Keep empty containers together if we can. - lbracket = node.children[0] - rbracket = node.children[-1] - if len(node.children) == 2: - self._SetUnbreakable(node.children[-1]) - elif (rbracket.value in ']}' and - lbracket.get_lineno() == rbracket.get_lineno() and - rbracket.column - lbracket.column < style.Get('COLUMN_LIMIT')): - self._SetStronglyConnected(*node.children[1:]) - - ############################################################################ - # Helper methods that set the annotations. - - def _SetUnbreakable(self, node): - """Set an UNBREAKABLE penalty annotation for the given node.""" - _RecAnnotate(node, pytree_utils.Annotation.SPLIT_PENALTY, UNBREAKABLE) - - def _SetStronglyConnected(self, *nodes): - """Set a STRONGLY_CONNECTED penalty annotation for the given nodes.""" - for node in nodes: - _RecAnnotate(node, pytree_utils.Annotation.SPLIT_PENALTY, - STRONGLY_CONNECTED) - - def _SetUnbreakableOnChildren(self, node): - """Set an UNBREAKABLE penalty annotation on children of node.""" - for child in node.children: - self.Visit(child) - start = 2 if hasattr(node.children[0], 'is_pseudo') else 1 - for i in py3compat.range(start, len(node.children)): - self._SetUnbreakable(node.children[i]) - - def _SetExpressionPenalty(self, node, penalty): - """Set an ARITHMETIC_EXPRESSION penalty annotation children nodes.""" - - def RecArithmeticExpression(node, first_child_leaf): - if node is first_child_leaf: - return - - if isinstance(node, pytree.Leaf): - if node.value in {'(', 'for', 'if'}: - return - penalty_annotation = pytree_utils.GetNodeAnnotation( - node, pytree_utils.Annotation.SPLIT_PENALTY, default=0) - if penalty_annotation < penalty: - pytree_utils.SetNodeAnnotation(node, - pytree_utils.Annotation.SPLIT_PENALTY, - penalty) - else: - for child in node.children: - RecArithmeticExpression(child, first_child_leaf) - - RecArithmeticExpression(node, _FirstChildNode(node)) - - -def _RecAnnotate(tree, annotate_name, annotate_value): - """Recursively set the given annotation on all leafs of the subtree. - - Takes care to only increase the penalty. If the node already has a higher - or equal penalty associated with it, this is a no-op. - - Args: - tree: subtree to annotate - annotate_name: name of the annotation to set - annotate_value: value of the annotation to set - """ - for child in tree.children: - _RecAnnotate(child, annotate_name, annotate_value) - if isinstance(tree, pytree.Leaf): - cur_annotate = pytree_utils.GetNodeAnnotation( - tree, annotate_name, default=0) - if cur_annotate < annotate_value: - pytree_utils.SetNodeAnnotation(tree, annotate_name, annotate_value) - - -def _AllowBuilderStyleCalls(node): - """Allow splitting before '.' if it's a builder style function call.""" - - def RecGetLeaves(node): - if isinstance(node, pytree.Leaf): - return [node] - children = [] - for child in node.children: - children += RecGetLeaves(child) - return children - - list_of_children = RecGetLeaves(node) - prev_child = None - for child in list_of_children: - if child.value == '.': - if prev_child.lineno != child.lineno: - pytree_utils.SetNodeAnnotation(child, - pytree_utils.Annotation.SPLIT_PENALTY, 0) - prev_child = child - - -def _FirstChildNode(node): - if isinstance(node, pytree.Leaf): - return node - return _FirstChildNode(node.children[0]) - - -def _LastChildNode(node): - if isinstance(node, pytree.Leaf): - return node - return _LastChildNode(node.children[-1]) +# Generic split penalties +UNBREAKABLE = 1000**5 +VERY_STRONGLY_CONNECTED = 5000 +STRONGLY_CONNECTED = 2500 + +############################################################################# +# Grammar-specific penalties - should be <= 1000 # +############################################################################# + +# Lambdas shouldn't be split unless absolutely necessary or if +# ALLOW_MULTILINE_LAMBDAS is True. +LAMBDA = 1000 +MULTILINE_LAMBDA = 500 + +ANNOTATION = 100 +ARGUMENT = 25 + +# TODO: Assign real values. +RETURN_TYPE = 1 +DOTTED_NAME = 40 +EXPR = 10 +DICT_KEY_EXPR = 20 +DICT_VALUE_EXPR = 11 diff --git a/yapf/yapflib/style.py b/yapf/yapflib/style.py index ae7333ceb..cbe0eb16b 100644 --- a/yapf/yapflib/style.py +++ b/yapf/yapflib/style.py @@ -1,4 +1,4 @@ -# Copyright 2015-2016 Google Inc. All Rights Reserved. +# Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,10 +15,16 @@ import os import re +import sys import textwrap +from configparser import ConfigParser from yapf.yapflib import errors -from yapf.yapflib import py3compat + +if sys.version_info >= (3, 11): + import tomllib +else: + import tomli as tomllib class StyleConfigError(errors.YapfError): @@ -31,6 +37,11 @@ def Get(setting_name): return _style[setting_name] +def GetOrDefault(setting_name, default_value): + """Get a style setting or default value if the setting does not exist.""" + return _style.get(setting_name, default_value) + + def Help(): """Return dict mapping style names to help strings.""" return _STYLE_HELP @@ -39,14 +50,110 @@ def Help(): def SetGlobalStyle(style): """Set a style dict.""" global _style + global _GLOBAL_STYLE_FACTORY + factory = _GetStyleFactory(style) + if factory: + _GLOBAL_STYLE_FACTORY = factory _style = style _STYLE_HELP = dict( + # BASED_ON_STYLE='Which predefined style this style is based on', + ALIGN_ASSIGNMENT=textwrap.dedent("""\ + Align assignment or augmented assignment operators. + If there is a blank line or newline comment or objects with newline entries in between, + it will start new block alignment. For exemple: + + val_first = 1 + val_second += 2 + object = { + entry1:1, + entry2:2, + entry3:3, + } + val_third = 3 + + will be formatted as: + + val_first = 1 + val_second += 2 + object = { + entry1: 1, + entry2: 2, + entry3: 3, + } + val_third = 3 + """), + ALIGN_ASSIGNMENT_RESTART_AFTER_COMMENTS=textwrap.dedent("""\ + Start new assignment alignment block when there is a newline comment in + between. For example if this is off then: + + val_first = 1 + val_second += 2 + # comment + val_third = 3 + + will be formatted as: + + val_first = 1 + val_second += 2 + # comment + val_third = 3 + + and if it is on it will be formatted as: + + val_first = 1 + val_second += 2 + # comment + val_third = 3 + """), ALIGN_CLOSING_BRACKET_WITH_VISUAL_INDENT=textwrap.dedent("""\ - Align closing bracket with visual indentation."""), + Align closing bracket with visual indentation. + """), + ALLOW_MULTILINE_DICTIONARY_KEYS=textwrap.dedent("""\ + Allow dictionary keys to exist on multiple lines. For example: + + x = { + ('this is the first element of a tuple', + 'this is the second element of a tuple'): + value, + } + """), ALLOW_MULTILINE_LAMBDAS=textwrap.dedent("""\ - Allow lambdas to be formatted on more than one line."""), + Allow lambdas to be formatted on more than one line. + """), + ALLOW_SPLIT_BEFORE_DEFAULT_OR_NAMED_ASSIGNS=textwrap.dedent("""\ + Allow splitting before a default / named assignment in an argument list. + """), + ALLOW_SPLIT_BEFORE_DICT_VALUE=textwrap.dedent("""\ + Allow splits before the dictionary value. + """), + ARITHMETIC_PRECEDENCE_INDICATION=textwrap.dedent("""\ + Let spacing indicate operator precedence. For example: + + a = 1 * 2 + 3 / 4 + b = 1 / 2 - 3 * 4 + c = (1 + 2) * (3 - 4) + d = (1 - 2) / (3 + 4) + e = 1 * 2 - 3 + f = 1 + 2 + 3 + 4 + + will be formatted as follows to indicate precedence: + + a = 1*2 + 3/4 + b = 1/2 - 3*4 + c = (1+2) * (3-4) + d = (1-2) / (3+4) + e = 1*2 - 3 + f = 1 + 2 + 3 + 4 + + """), + BLANK_LINE_BEFORE_CLASS_DOCSTRING=textwrap.dedent("""\ + Insert a blank line before a class-level docstring. + """), + BLANK_LINE_BEFORE_MODULE_DOCSTRING=textwrap.dedent("""\ + Insert a blank line before a module docstring. + """), BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF=textwrap.dedent("""\ Insert a blank line before a 'def' or 'class' immediately nested within another 'def' or 'class'. For example: @@ -54,7 +161,16 @@ def SetGlobalStyle(style): class Foo: # <------ this blank line def method(): - ..."""), + pass + """), + BLANK_LINES_AROUND_TOP_LEVEL_DEFINITION=textwrap.dedent("""\ + Number of blank lines surrounding top-level function and class + definitions. + """), + BLANK_LINES_BETWEEN_TOP_LEVEL_IMPORTS_AND_VARIABLES=textwrap.dedent("""\ + Number of blank lines between top-level imports and variable + definitions. + """), COALESCE_BRACKETS=textwrap.dedent("""\ Do not split consecutive brackets. Only relevant when dedent_closing_brackets is set. For example: @@ -71,11 +187,25 @@ def method(): call_func_that_takes_a_dict({ 'key1': 'value1', 'key2': 'value2', - })"""), + }) + """), COLUMN_LIMIT=textwrap.dedent("""\ - The column limit."""), + The column limit. + """), + CONTINUATION_ALIGN_STYLE=textwrap.dedent("""\ + The style for continuation alignment. Possible values are: + + - SPACE: Use spaces for continuation alignment. This is default behavior. + - FIXED: Use fixed number (CONTINUATION_INDENT_WIDTH) of columns + (ie: CONTINUATION_INDENT_WIDTH/INDENT_WIDTH tabs or + CONTINUATION_INDENT_WIDTH spaces) for continuation alignment. + - VALIGN-RIGHT: Vertically align continuation lines to multiple of + INDENT_WIDTH columns. Slightly right (one tab or a few spaces) if + cannot vertically align continuation lines with indent characters. + """), CONTINUATION_INDENT_WIDTH=textwrap.dedent("""\ - Indent width used for line continuations."""), + Indent width used for line continuations. + """), DEDENT_CLOSING_BRACKETS=textwrap.dedent("""\ Put closing brackets on a separate line, dedented, if the bracketed expression can't fit in a single line. Applies to all kinds of brackets, @@ -92,15 +222,61 @@ def method(): transform=Transformation.AVERAGE(window=timedelta(seconds=60)), start_ts=now()-timedelta(days=3), end_ts=now(), - ) # <--- this bracket is dedented and on a separate line"""), + ) # <--- this bracket is dedented and on a separate line + """), + DISABLE_ENDING_COMMA_HEURISTIC=textwrap.dedent("""\ + Disable the heuristic which places each list element on a separate line + if the list is comma-terminated. + + Note: The behavior of this flag changed in v0.40.3. Before, if this flag + was true, we would split lists that contained a trailing comma or a + comment. Now, we have a separate flag, `DISABLE_SPLIT_LIT_WITH_COMMENT`, + that controls splitting when a list contains a comment. To get the old + behavior, set both flags to true. More information in CHANGELOG.md. + """), + DISABLE_SPLIT_LIST_WITH_COMMENT=textwrap.dedent(""" + Don't put every element on a new line within a list that contains + interstitial comments. + """), + EACH_DICT_ENTRY_ON_SEPARATE_LINE=textwrap.dedent("""\ + Place each dictionary entry onto its own line. + """), + FORCE_MULTILINE_DICT=textwrap.dedent("""\ + Require multiline dictionary even if it would normally fit on one line. + For example: + + config = { + 'key1': 'value1' + } + """), I18N_COMMENT=textwrap.dedent("""\ The regex for an i18n comment. The presence of this comment stops reformatting of that line, because the comments are required to be - next to the string they translate."""), + next to the string they translate. + """), I18N_FUNCTION_CALL=textwrap.dedent("""\ The i18n function call names. The presence of this function stops reformattting on that line, because the string it has cannot be moved - away from the i18n comment."""), + away from the i18n comment. + """), + INDENT_CLOSING_BRACKETS=textwrap.dedent("""\ + Put closing brackets on a separate line, indented, if the bracketed + expression can't fit in a single line. Applies to all kinds of brackets, + including function definitions and calls. For example: + + config = { + 'key1': 'value1', + 'key2': 'value2', + } # <--- this bracket is indented and on a separate line + + time_series = self.remote_client.query_entity_counters( + entity='dev3246.region1', + key='dns.query_latency_tcp', + transform=Transformation.AVERAGE(window=timedelta(seconds=60)), + start_ts=now()-timedelta(days=3), + end_ts=now(), + ) # <--- this bracket is indented and on a separate line + """), INDENT_DICTIONARY_VALUE=textwrap.dedent("""\ Indent the dictionary value if it cannot fit on the same line as the dictionary key. For example: @@ -110,46 +286,220 @@ def method(): 'value1', 'key2': value1 + value2, - }"""), + } + """), + INDENT_BLANK_LINES=textwrap.dedent("""\ + Indent blank lines. + """), INDENT_WIDTH=textwrap.dedent("""\ - The number of columns to use for indentation."""), + The number of columns to use for indentation. + """), JOIN_MULTIPLE_LINES=textwrap.dedent("""\ - Join short lines into one line. E.g., single line 'if' statements."""), - SPACES_AROUND_POWER_OPERATOR=textwrap.dedent("""\ - Use spaces around the power operator."""), - SPACES_BEFORE_COMMENT=textwrap.dedent("""\ - The number of spaces required before a trailing comment."""), + Join short lines into one line. E.g., single line 'if' statements. + """), + NO_SPACES_AROUND_SELECTED_BINARY_OPERATORS=textwrap.dedent("""\ + Do not include spaces around selected binary operators. For example: + + 1 + 2 * 3 - 4 / 5 + + will be formatted as follows when configured with "*,/": + + 1 + 2*3 - 4/5 + """), SPACE_BETWEEN_ENDING_COMMA_AND_CLOSING_BRACKET=textwrap.dedent("""\ Insert a space between the ending comma and closing bracket of a list, - etc."""), + etc. + """), + SPACE_INSIDE_BRACKETS=textwrap.dedent("""\ + Use spaces inside brackets, braces, and parentheses. For example: + + method_call( 1 ) + my_dict[ 3 ][ 1 ][ get_index( *args, **kwargs ) ] + my_set = { 1, 2, 3 } + """), + SPACES_AROUND_DEFAULT_OR_NAMED_ASSIGN=textwrap.dedent("""\ + Use spaces around default or named assigns. + """), + SPACES_AROUND_DICT_DELIMITERS=textwrap.dedent("""\ + Adds a space after the opening '{' and before the ending '}' dict + delimiters. + + {1: 2} + + will be formatted as: + + { 1: 2 } + """), + SPACES_AROUND_LIST_DELIMITERS=textwrap.dedent("""\ + Adds a space after the opening '[' and before the ending ']' list + delimiters. + + [1, 2] + + will be formatted as: + + [ 1, 2 ] + """), + SPACES_AROUND_POWER_OPERATOR=textwrap.dedent("""\ + Use spaces around the power operator. + """), + SPACES_AROUND_SUBSCRIPT_COLON=textwrap.dedent("""\ + Use spaces around the subscript / slice operator. For example: + + my_list[1 : 10 : 2] + """), + SPACES_AROUND_TUPLE_DELIMITERS=textwrap.dedent("""\ + Adds a space after the opening '(' and before the ending ')' tuple + delimiters. + + (1, 2, 3) + + will be formatted as: + + ( 1, 2, 3 ) + """), + SPACES_BEFORE_COMMENT=textwrap.dedent("""\ + The number of spaces required before a trailing comment. + This can be a single value (representing the number of spaces + before each trailing comment) or list of values (representing + alignment column values; trailing comments within a block will + be aligned to the first column value that is greater than the maximum + line length within the block). For example: + + With spaces_before_comment=5: + + 1 + 1 # Adding values + + will be formatted as: + + 1 + 1 # Adding values <-- 5 spaces between the end of the + # statement and comment + + With spaces_before_comment=15, 20: + + 1 + 1 # Adding values + two + two # More adding + + longer_statement # This is a longer statement + short # This is a shorter statement + + a_very_long_statement_that_extends_beyond_the_final_column # Comment + short # This is a shorter statement + + will be formatted as: + + 1 + 1 # Adding values <-- end of line comments in block + # aligned to col 15 + two + two # More adding + + longer_statement # This is a longer statement <-- end of line + # comments in block aligned to col 20 + short # This is a shorter statement + + a_very_long_statement_that_extends_beyond_the_final_column # Comment <-- the end of line comments are aligned based on the line length + short # This is a shorter statement + + """), # noqa + SPLIT_ALL_COMMA_SEPARATED_VALUES=textwrap.dedent("""\ + Split before arguments. + """), + SPLIT_ALL_TOP_LEVEL_COMMA_SEPARATED_VALUES=textwrap.dedent("""\ + Split before arguments, but do not split all subexpressions recursively + (unless needed). + """), SPLIT_ARGUMENTS_WHEN_COMMA_TERMINATED=textwrap.dedent("""\ Split before arguments if the argument list is terminated by a - comma."""), + comma. + """), + SPLIT_BEFORE_ARITHMETIC_OPERATOR=textwrap.dedent("""\ + Set to True to prefer splitting before '+', '-', '*', '/', '//', or '@' + rather than after. + """), SPLIT_BEFORE_BITWISE_OPERATOR=textwrap.dedent("""\ Set to True to prefer splitting before '&', '|' or '^' rather than - after."""), + after. + """), + SPLIT_BEFORE_CLOSING_BRACKET=textwrap.dedent("""\ + Split before the closing bracket if a list or dict literal doesn't fit on + a single line. + """), + SPLIT_BEFORE_DICT_SET_GENERATOR=textwrap.dedent("""\ + Split before a dictionary or set generator (comp_for). For example, note + the split before the 'for': + + foo = { + variable: 'Hello world, have a nice day!' + for variable in bar if variable != 42 + } + """), + SPLIT_BEFORE_DOT=textwrap.dedent("""\ + Split before the '.' if we need to split a longer expression: + + foo = ('This is a really long string: {}, {}, {}, {}'.format(a, b, c, d)) + + would reformat to something like: + + foo = ('This is a really long string: {}, {}, {}, {}' + .format(a, b, c, d)) + """), # noqa + SPLIT_BEFORE_EXPRESSION_AFTER_OPENING_PAREN=textwrap.dedent("""\ + Split after the opening paren which surrounds an expression if it doesn't + fit on a single line. + """), SPLIT_BEFORE_FIRST_ARGUMENT=textwrap.dedent("""\ If an argument / parameter list is going to be split, then split before - the first argument."""), + the first argument. + """), SPLIT_BEFORE_LOGICAL_OPERATOR=textwrap.dedent("""\ Set to True to prefer splitting before 'and' or 'or' rather than - after."""), + after. + """), SPLIT_BEFORE_NAMED_ASSIGNS=textwrap.dedent("""\ - Split named assignments onto individual lines."""), + Split named assignments onto individual lines. + """), + SPLIT_COMPLEX_COMPREHENSION=textwrap.dedent("""\ + Set to True to split list comprehensions and generators that have + non-trivial expressions and multiple clauses before each of these + clauses. For example: + + result = [ + a_long_var + 100 for a_long_var in xrange(1000) + if a_long_var % 10] + + would reformat to something like: + + result = [ + a_long_var + 100 + for a_long_var in xrange(1000) + if a_long_var % 10] + """), SPLIT_PENALTY_AFTER_OPENING_BRACKET=textwrap.dedent("""\ - The penalty for splitting right after the opening bracket."""), + The penalty for splitting right after the opening bracket. + """), SPLIT_PENALTY_AFTER_UNARY_OPERATOR=textwrap.dedent("""\ - The penalty for splitting the line after a unary operator."""), + The penalty for splitting the line after a unary operator. + """), + SPLIT_PENALTY_ARITHMETIC_OPERATOR=textwrap.dedent("""\ + The penalty of splitting the line around the '+', '-', '*', '/', '//', + `%`, and '@' operators. + """), SPLIT_PENALTY_BEFORE_IF_EXPR=textwrap.dedent("""\ - The penalty for splitting right before an if expression."""), + The penalty for splitting right before an if expression. + """), SPLIT_PENALTY_BITWISE_OPERATOR=textwrap.dedent("""\ - The penalty of splitting the line around the '&', '|', and '^' - operators."""), + The penalty of splitting the line around the '&', '|', and '^' operators. + """), + SPLIT_PENALTY_COMPREHENSION=textwrap.dedent("""\ + The penalty for splitting a list comprehension or generator + expression. + """), SPLIT_PENALTY_EXCESS_CHARACTER=textwrap.dedent("""\ - The penalty for characters over the column limit."""), + The penalty for characters over the column limit. + """), SPLIT_PENALTY_FOR_ADDED_LINE_SPLIT=textwrap.dedent("""\ - The penalty incurred by adding a line split to the unwrapped line. The - more line splits added the higher the penalty."""), + The penalty incurred by adding a line split to the logical line. The + more line splits added the higher the penalty. + """), SPLIT_PENALTY_IMPORT_NAMES=textwrap.dedent("""\ The penalty of splitting a list of "import as" names. For example: @@ -161,91 +511,167 @@ def method(): from a_very_long_or_indented_module_name_yada_yad import ( long_argument_1, long_argument_2, long_argument_3) - """), + """), # noqa SPLIT_PENALTY_LOGICAL_OPERATOR=textwrap.dedent("""\ - The penalty of splitting the line around the 'and' and 'or' - operators."""), + The penalty of splitting the line around the 'and' and 'or' operators. + """), USE_TABS=textwrap.dedent("""\ - Use the Tab character for indentation."""), - # BASED_ON_STYLE='Which predefined style this style is based on', + Use the Tab character for indentation. + """), ) def CreatePEP8Style(): + """Create the PEP8 formatting style.""" return dict( + ALIGN_ASSIGNMENT=False, + ALIGN_ASSIGNMENT_RESTART_AFTER_COMMENTS=False, ALIGN_CLOSING_BRACKET_WITH_VISUAL_INDENT=True, + ALLOW_MULTILINE_DICTIONARY_KEYS=False, ALLOW_MULTILINE_LAMBDAS=False, - COLUMN_LIMIT=79, + ALLOW_SPLIT_BEFORE_DEFAULT_OR_NAMED_ASSIGNS=True, + ALLOW_SPLIT_BEFORE_DICT_VALUE=True, + ARITHMETIC_PRECEDENCE_INDICATION=False, + BLANK_LINE_BEFORE_CLASS_DOCSTRING=False, + BLANK_LINE_BEFORE_MODULE_DOCSTRING=False, + BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF=True, + BLANK_LINES_AROUND_TOP_LEVEL_DEFINITION=2, + BLANK_LINES_BETWEEN_TOP_LEVEL_IMPORTS_AND_VARIABLES=1, COALESCE_BRACKETS=False, + COLUMN_LIMIT=79, + CONTINUATION_ALIGN_STYLE='SPACE', + CONTINUATION_INDENT_WIDTH=4, DEDENT_CLOSING_BRACKETS=False, + DISABLE_ENDING_COMMA_HEURISTIC=False, + DISABLE_SPLIT_LIST_WITH_COMMENT=False, + EACH_DICT_ENTRY_ON_SEPARATE_LINE=True, + FORCE_MULTILINE_DICT=False, I18N_COMMENT='', I18N_FUNCTION_CALL='', + INDENT_CLOSING_BRACKETS=False, INDENT_DICTIONARY_VALUE=False, INDENT_WIDTH=4, - CONTINUATION_INDENT_WIDTH=4, - BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF=False, + INDENT_BLANK_LINES=False, JOIN_MULTIPLE_LINES=True, + NO_SPACES_AROUND_SELECTED_BINARY_OPERATORS=set(), SPACE_BETWEEN_ENDING_COMMA_AND_CLOSING_BRACKET=True, + SPACE_INSIDE_BRACKETS=False, + SPACES_AROUND_DEFAULT_OR_NAMED_ASSIGN=False, + SPACES_AROUND_DICT_DELIMITERS=False, + SPACES_AROUND_LIST_DELIMITERS=False, SPACES_AROUND_POWER_OPERATOR=False, + SPACES_AROUND_SUBSCRIPT_COLON=False, + SPACES_AROUND_TUPLE_DELIMITERS=False, SPACES_BEFORE_COMMENT=2, + SPLIT_ALL_COMMA_SEPARATED_VALUES=False, + SPLIT_ALL_TOP_LEVEL_COMMA_SEPARATED_VALUES=False, SPLIT_ARGUMENTS_WHEN_COMMA_TERMINATED=False, - SPLIT_BEFORE_BITWISE_OPERATOR=False, + SPLIT_BEFORE_ARITHMETIC_OPERATOR=False, + SPLIT_BEFORE_BITWISE_OPERATOR=True, + SPLIT_BEFORE_CLOSING_BRACKET=True, + SPLIT_BEFORE_DICT_SET_GENERATOR=True, + SPLIT_BEFORE_DOT=False, + SPLIT_BEFORE_EXPRESSION_AFTER_OPENING_PAREN=False, SPLIT_BEFORE_FIRST_ARGUMENT=False, - SPLIT_BEFORE_LOGICAL_OPERATOR=False, + SPLIT_BEFORE_LOGICAL_OPERATOR=True, SPLIT_BEFORE_NAMED_ASSIGNS=True, + SPLIT_COMPLEX_COMPREHENSION=False, + SPLIT_PENALTY_AFTER_OPENING_BRACKET=300, SPLIT_PENALTY_AFTER_UNARY_OPERATOR=10000, - SPLIT_PENALTY_EXCESS_CHARACTER=2600, + SPLIT_PENALTY_ARITHMETIC_OPERATOR=300, + SPLIT_PENALTY_BEFORE_IF_EXPR=0, SPLIT_PENALTY_BITWISE_OPERATOR=300, + SPLIT_PENALTY_COMPREHENSION=80, + SPLIT_PENALTY_EXCESS_CHARACTER=7000, + SPLIT_PENALTY_FOR_ADDED_LINE_SPLIT=30, SPLIT_PENALTY_IMPORT_NAMES=0, SPLIT_PENALTY_LOGICAL_OPERATOR=300, - SPLIT_PENALTY_AFTER_OPENING_BRACKET=30, - SPLIT_PENALTY_FOR_ADDED_LINE_SPLIT=30, - SPLIT_PENALTY_BEFORE_IF_EXPR=0, - USE_TABS=False - ) # yapf: disable + USE_TABS=False, + ) def CreateGoogleStyle(): + """Create the Google formatting style.""" style = CreatePEP8Style() style['ALIGN_CLOSING_BRACKET_WITH_VISUAL_INDENT'] = False style['COLUMN_LIMIT'] = 80 + style['INDENT_DICTIONARY_VALUE'] = True style['INDENT_WIDTH'] = 4 - style['BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF'] = True style['I18N_COMMENT'] = r'#\..*' style['I18N_FUNCTION_CALL'] = ['N_', '_'] + style['JOIN_MULTIPLE_LINES'] = False style['SPACE_BETWEEN_ENDING_COMMA_AND_CLOSING_BRACKET'] = False + style['SPLIT_BEFORE_BITWISE_OPERATOR'] = False + style['SPLIT_BEFORE_DICT_SET_GENERATOR'] = False + style['SPLIT_BEFORE_LOGICAL_OPERATOR'] = False + style['SPLIT_COMPLEX_COMPREHENSION'] = True + style['SPLIT_PENALTY_COMPREHENSION'] = 2100 return style -def CreateChromiumStyle(): +def CreateYapfStyle(): + """Create the YAPF formatting style.""" style = CreateGoogleStyle() - style['INDENT_DICTIONARY_VALUE'] = True + style['ALLOW_MULTILINE_DICTIONARY_KEYS'] = True + style['ALLOW_SPLIT_BEFORE_DEFAULT_OR_NAMED_ASSIGNS'] = False style['INDENT_WIDTH'] = 2 - style['JOIN_MULTIPLE_LINES'] = False style['SPLIT_BEFORE_BITWISE_OPERATOR'] = True - style['SPLIT_PENALTY_FOR_ADDED_LINE_SPLIT'] = 0 + style['SPLIT_BEFORE_DOT'] = True + style['SPLIT_BEFORE_EXPRESSION_AFTER_OPENING_PAREN'] = True return style def CreateFacebookStyle(): + """Create the Facebook formatting style.""" style = CreatePEP8Style() style['ALIGN_CLOSING_BRACKET_WITH_VISUAL_INDENT'] = False + style['BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF'] = False style['COLUMN_LIMIT'] = 80 style['DEDENT_CLOSING_BRACKETS'] = True + style['INDENT_CLOSING_BRACKETS'] = False + style['INDENT_DICTIONARY_VALUE'] = True style['JOIN_MULTIPLE_LINES'] = False style['SPACES_BEFORE_COMMENT'] = 2 style['SPLIT_PENALTY_AFTER_OPENING_BRACKET'] = 0 - style['SPLIT_PENALTY_FOR_ADDED_LINE_SPLIT'] = 30 style['SPLIT_PENALTY_BEFORE_IF_EXPR'] = 30 + style['SPLIT_PENALTY_FOR_ADDED_LINE_SPLIT'] = 30 + style['SPLIT_BEFORE_BITWISE_OPERATOR'] = False + style['SPLIT_BEFORE_LOGICAL_OPERATOR'] = False return style _STYLE_NAME_TO_FACTORY = dict( - pep8=CreatePEP8Style, - chromium=CreateChromiumStyle, - google=CreateGoogleStyle, facebook=CreateFacebookStyle, -) # yapf: disable + google=CreateGoogleStyle, + pep8=CreatePEP8Style, + yapf=CreateYapfStyle, +) + +_DEFAULT_STYLE_TO_FACTORY = [ + (CreateFacebookStyle(), CreateFacebookStyle), + (CreateGoogleStyle(), CreateGoogleStyle), + (CreatePEP8Style(), CreatePEP8Style), + (CreateYapfStyle(), CreateYapfStyle), +] + + +def _GetStyleFactory(style): + for def_style, factory in _DEFAULT_STYLE_TO_FACTORY: + if style == def_style: + return factory + return None + + +def _ContinuationAlignStyleStringConverter(s): + """Option value converter for a continuation align style string.""" + accepted_styles = ('SPACE', 'FIXED', 'VALIGN-RIGHT') + if s: + r = s.strip('"\'').replace('_', '-').upper() + if r not in accepted_styles: + raise ValueError('unknown continuation align style: %r' % (s,)) + else: + r = accepted_styles[0] + return r def _StringListConverter(s): @@ -253,9 +679,33 @@ def _StringListConverter(s): return [part.strip() for part in s.split(',')] +def _StringSetConverter(s): + """Option value converter for a comma-separated set of strings.""" + if len(s) > 2 and s[0] in '"\'': + s = s[1:-1] + return {part.strip() for part in s.split(',')} + + def _BoolConverter(s): """Option value converter for a boolean.""" - return py3compat.CONFIGPARSER_BOOLEAN_STATES[s.lower()] + return ConfigParser.BOOLEAN_STATES[s.lower()] + + +def _IntListConverter(s): + """Option value converter for a comma-separated list of integers.""" + s = s.strip() + if s.startswith('[') and s.endswith(']'): + s = s[1:-1] + + return [int(part.strip()) for part in s.split(',') if part.strip()] + + +def _IntOrIntListConverter(s): + """Option value converter for an integer or list of integers.""" + if len(s) > 2 and s[0] in '"\'': + s = s[1:-1] + return _IntListConverter(s) if ',' in s else int(s) + # Different style options need to have their values interpreted differently when # read from the config file. This dict maps an option name to a "converter" @@ -265,36 +715,70 @@ def _BoolConverter(s): # # Note: this dict has to map all the supported style options. _STYLE_OPTION_VALUE_CONVERTER = dict( + ALIGN_ASSIGNMENT=_BoolConverter, + ALIGN_ASSIGNMENT_RESTART_AFTER_COMMENTS=_BoolConverter, ALIGN_CLOSING_BRACKET_WITH_VISUAL_INDENT=_BoolConverter, + ALLOW_MULTILINE_DICTIONARY_KEYS=_BoolConverter, ALLOW_MULTILINE_LAMBDAS=_BoolConverter, + ALLOW_SPLIT_BEFORE_DEFAULT_OR_NAMED_ASSIGNS=_BoolConverter, + ALLOW_SPLIT_BEFORE_DICT_VALUE=_BoolConverter, + ARITHMETIC_PRECEDENCE_INDICATION=_BoolConverter, + BLANK_LINE_BEFORE_CLASS_DOCSTRING=_BoolConverter, + BLANK_LINE_BEFORE_MODULE_DOCSTRING=_BoolConverter, + BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF=_BoolConverter, + BLANK_LINES_AROUND_TOP_LEVEL_DEFINITION=int, + BLANK_LINES_BETWEEN_TOP_LEVEL_IMPORTS_AND_VARIABLES=int, COALESCE_BRACKETS=_BoolConverter, COLUMN_LIMIT=int, + CONTINUATION_ALIGN_STYLE=_ContinuationAlignStyleStringConverter, + CONTINUATION_INDENT_WIDTH=int, DEDENT_CLOSING_BRACKETS=_BoolConverter, + DISABLE_ENDING_COMMA_HEURISTIC=_BoolConverter, + DISABLE_SPLIT_LIST_WITH_COMMENT=_BoolConverter, + EACH_DICT_ENTRY_ON_SEPARATE_LINE=_BoolConverter, + FORCE_MULTILINE_DICT=_BoolConverter, I18N_COMMENT=str, I18N_FUNCTION_CALL=_StringListConverter, + INDENT_BLANK_LINES=_BoolConverter, + INDENT_CLOSING_BRACKETS=_BoolConverter, INDENT_DICTIONARY_VALUE=_BoolConverter, INDENT_WIDTH=int, - CONTINUATION_INDENT_WIDTH=int, - BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF=_BoolConverter, JOIN_MULTIPLE_LINES=_BoolConverter, + NO_SPACES_AROUND_SELECTED_BINARY_OPERATORS=_StringSetConverter, SPACE_BETWEEN_ENDING_COMMA_AND_CLOSING_BRACKET=_BoolConverter, + SPACE_INSIDE_BRACKETS=_BoolConverter, + SPACES_AROUND_DEFAULT_OR_NAMED_ASSIGN=_BoolConverter, + SPACES_AROUND_DICT_DELIMITERS=_BoolConverter, + SPACES_AROUND_LIST_DELIMITERS=_BoolConverter, SPACES_AROUND_POWER_OPERATOR=_BoolConverter, - SPACES_BEFORE_COMMENT=int, + SPACES_AROUND_SUBSCRIPT_COLON=_BoolConverter, + SPACES_AROUND_TUPLE_DELIMITERS=_BoolConverter, + SPACES_BEFORE_COMMENT=_IntOrIntListConverter, + SPLIT_ALL_COMMA_SEPARATED_VALUES=_BoolConverter, + SPLIT_ALL_TOP_LEVEL_COMMA_SEPARATED_VALUES=_BoolConverter, SPLIT_ARGUMENTS_WHEN_COMMA_TERMINATED=_BoolConverter, + SPLIT_BEFORE_ARITHMETIC_OPERATOR=_BoolConverter, SPLIT_BEFORE_BITWISE_OPERATOR=_BoolConverter, + SPLIT_BEFORE_CLOSING_BRACKET=_BoolConverter, + SPLIT_BEFORE_DICT_SET_GENERATOR=_BoolConverter, + SPLIT_BEFORE_DOT=_BoolConverter, + SPLIT_BEFORE_EXPRESSION_AFTER_OPENING_PAREN=_BoolConverter, SPLIT_BEFORE_FIRST_ARGUMENT=_BoolConverter, SPLIT_BEFORE_LOGICAL_OPERATOR=_BoolConverter, SPLIT_BEFORE_NAMED_ASSIGNS=_BoolConverter, + SPLIT_COMPLEX_COMPREHENSION=_BoolConverter, + SPLIT_PENALTY_AFTER_OPENING_BRACKET=int, SPLIT_PENALTY_AFTER_UNARY_OPERATOR=int, - SPLIT_PENALTY_EXCESS_CHARACTER=int, + SPLIT_PENALTY_ARITHMETIC_OPERATOR=int, + SPLIT_PENALTY_BEFORE_IF_EXPR=int, SPLIT_PENALTY_BITWISE_OPERATOR=int, + SPLIT_PENALTY_COMPREHENSION=int, + SPLIT_PENALTY_EXCESS_CHARACTER=int, + SPLIT_PENALTY_FOR_ADDED_LINE_SPLIT=int, SPLIT_PENALTY_IMPORT_NAMES=int, SPLIT_PENALTY_LOGICAL_OPERATOR=int, - SPLIT_PENALTY_AFTER_OPENING_BRACKET=int, - SPLIT_PENALTY_FOR_ADDED_LINE_SPLIT=int, - SPLIT_PENALTY_BEFORE_IF_EXPR=int, - USE_TABS=_BoolConverter -) # yapf: disable + USE_TABS=_BoolConverter, +) def CreateStyleFromConfig(style_config): @@ -304,7 +788,7 @@ def CreateStyleFromConfig(style_config): style_config: either a style name or a file name. The file is expected to contain settings. It can have a special BASED_ON_STYLE setting naming the style which it derives from. If no such setting is found, it derives from - the default style. When style_config is None, the DEFAULT_STYLE_FACTORY + the default style. When style_config is None, the _GLOBAL_STYLE_FACTORY config is created. Returns: @@ -313,29 +797,57 @@ def CreateStyleFromConfig(style_config): Raises: StyleConfigError: if an unknown style option was encountered. """ + + def GlobalStyles(): + for style, _ in _DEFAULT_STYLE_TO_FACTORY: + yield style + + def_style = False if style_config is None: - return DEFAULT_STYLE_FACTORY() - style_factory = _STYLE_NAME_TO_FACTORY.get(style_config.lower()) - if style_factory is not None: - return style_factory() - if style_config.startswith('{'): - # Most likely a style specification from the command line. - config = _CreateConfigParserFromConfigString(style_config) - else: - # Unknown config name: assume it's a file name then. - config = _CreateConfigParserFromConfigFile(style_config) + for style in GlobalStyles(): + if _style == style: + def_style = True + break + if not def_style: + return _style + return _GLOBAL_STYLE_FACTORY() + + if isinstance(style_config, dict): + config = _CreateConfigParserFromConfigDict(style_config) + elif isinstance(style_config, str): + style_factory = _STYLE_NAME_TO_FACTORY.get(style_config.lower()) + if style_factory is not None: + return style_factory() + if style_config.startswith('{'): + # Most likely a style specification from the command line. + config = _CreateConfigParserFromConfigString(style_config) + else: + # Unknown config name: assume it's a file name then. + config = _CreateConfigParserFromConfigFile(style_config) return _CreateStyleFromConfigParser(config) +def _CreateConfigParserFromConfigDict(config_dict): + config = ConfigParser() + config.add_section('style') + for key, value in config_dict.items(): + config.set('style', key, str(value)) + return config + + def _CreateConfigParserFromConfigString(config_string): """Given a config string from the command line, return a config parser.""" if config_string[0] != '{' or config_string[-1] != '}': - raise StyleConfigError("Invalid style dict syntax: '{}'.".format( - config_string)) - config = py3compat.ConfigParser() + raise StyleConfigError( + "Invalid style dict syntax: '{}'.".format(config_string)) + config = ConfigParser() config.add_section('style') - for key, value in re.findall(r'([a-zA-Z0-9_]+)\s*[:=]\s*([a-zA-Z0-9_]+)', - config_string): + for key, value, _ in re.findall( + r'([a-zA-Z0-9_]+)\s*[:=]\s*' + r'(?:' + r'((?P[\'"]).*?(?P=quote)|' + r'[a-zA-Z0-9_]+)' + r')', config_string): # yapf: disable config.set('style', key, value) return config @@ -344,23 +856,40 @@ def _CreateConfigParserFromConfigFile(config_filename): """Read the file and return a ConfigParser object.""" if not os.path.exists(config_filename): # Provide a more meaningful error here. - raise StyleConfigError('"{0}" is not a valid style or file path'.format( - config_filename)) + raise StyleConfigError( + '"{0}" is not a valid style or file path'.format(config_filename)) + config = ConfigParser() + + if config_filename.endswith(PYPROJECT_TOML): + with open(config_filename, 'rb') as style_file: + pyproject_toml = tomllib.load(style_file) + style_dict = pyproject_toml.get('tool', {}).get('yapf', None) + if style_dict is None: + raise StyleConfigError( + 'Unable to find section [tool.yapf] in {0}'.format(config_filename)) + config.add_section('style') + for k, v in style_dict.items(): + config.set('style', k, str(v)) + return config + with open(config_filename) as style_file: - config = py3compat.ConfigParser() config.read_file(style_file) + if config_filename.endswith(SETUP_CONFIG): if not config.has_section('yapf'): - raise StyleConfigError('Unable to find section [yapf] in {0}'.format( - config_filename)) - elif config_filename.endswith(LOCAL_STYLE): - if not config.has_section('style'): - raise StyleConfigError('Unable to find section [style] in {0}'.format( - config_filename)) - else: + raise StyleConfigError( + 'Unable to find section [yapf] in {0}'.format(config_filename)) + return config + + if config_filename.endswith(LOCAL_STYLE): if not config.has_section('style'): - raise StyleConfigError('Unable to find section [style] in {0}'.format( - config_filename)) + raise StyleConfigError( + 'Unable to find section [style] in {0}'.format(config_filename)) + return config + + if not config.has_section('style'): + raise StyleConfigError( + 'Unable to find section [style] in {0}'.format(config_filename)) return config @@ -385,7 +914,7 @@ def _CreateStyleFromConfigParser(config): based_on = config.get('yapf', 'based_on_style').lower() base_style = _STYLE_NAME_TO_FACTORY[based_on]() else: - base_style = DEFAULT_STYLE_FACTORY() + base_style = _GLOBAL_STYLE_FACTORY() # Read all options specified in the file and update the style. for option, value in config.items(section): @@ -403,15 +932,18 @@ def _CreateStyleFromConfigParser(config): value, option)) return base_style + # The default style - used if yapf is not invoked without specifically # requesting a formatting style. DEFAULT_STYLE = 'pep8' DEFAULT_STYLE_FACTORY = CreatePEP8Style +_GLOBAL_STYLE_FACTORY = CreatePEP8Style # The name of the file to use for global style definition. -GLOBAL_STYLE = (os.path.join( - os.getenv('XDG_CONFIG_HOME') or os.path.expanduser('~/.config'), 'yapf', - 'style')) +GLOBAL_STYLE = ( + os.path.join( + os.getenv('XDG_CONFIG_HOME') or os.path.expanduser('~/.config'), 'yapf', + 'style')) # The name of the file to use for directory-local style definition. LOCAL_STYLE = '.style.yapf' @@ -420,8 +952,12 @@ def _CreateStyleFromConfigParser(config): # specified in the '[yapf]' section. SETUP_CONFIG = 'setup.cfg' +# Style definition by local pyproject.toml file. Style should be specified +# in the '[tool.yapf]' section. +PYPROJECT_TOML = 'pyproject.toml' + # TODO(eliben): For now we're preserving the global presence of a style dict. # Refactor this so that the style is passed around through yapf rather than # being global. -_style = {} -SetGlobalStyle(DEFAULT_STYLE_FACTORY()) +_style = None +SetGlobalStyle(_GLOBAL_STYLE_FACTORY()) diff --git a/yapf/yapflib/subtype_assigner.py b/yapf/yapflib/subtype_assigner.py deleted file mode 100644 index 4e70d613d..000000000 --- a/yapf/yapflib/subtype_assigner.py +++ /dev/null @@ -1,369 +0,0 @@ -# Copyright 2015-2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Subtype assigner for lib2to3 trees. - -This module assigns extra type information to the lib2to3 trees. This -information is more specific than whether something is an operator or an -identifier. For instance, it can specify if a node in the tree is part of a -subscript. - - AssignSubtypes(): the main function exported by this module. - -Annotations: - subtype: The subtype of a pytree token. See 'format_token' module for a list - of subtypes. -""" - -from lib2to3 import pytree -from lib2to3.pgen2 import token -from lib2to3.pygram import python_symbols as syms - -from yapf.yapflib import format_token -from yapf.yapflib import pytree_utils -from yapf.yapflib import pytree_visitor -from yapf.yapflib import style - - -def AssignSubtypes(tree): - """Run the subtype assigner visitor over the tree, modifying it in place. - - Arguments: - tree: the top-level pytree node to annotate with subtypes. - """ - subtype_assigner = _SubtypeAssigner() - subtype_assigner.Visit(tree) - -# Map tokens in argument lists to their respective subtype. -_ARGLIST_TOKEN_TO_SUBTYPE = { - '=': format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN, - '*': format_token.Subtype.VARARGS_STAR, - '**': format_token.Subtype.KWARGS_STAR_STAR, -} - - -class _SubtypeAssigner(pytree_visitor.PyTreeVisitor): - """_SubtypeAssigner - see file-level docstring for detailed description. - - The subtype is added as an annotation to the pytree token. - """ - - def Visit_dictsetmaker(self, node): # pylint: disable=invalid-name - # dictsetmaker ::= (test ':' test (comp_for | - # (',' test ':' test)* [','])) | - # (test (comp_for | (',' test)* [','])) - dict_maker = False - if len(node.children) > 1: - index = 0 - while index < len(node.children): - if node.children[index].type != token.COMMENT: - break - index += 1 - if index < len(node.children): - child = node.children[index + 1] - dict_maker = isinstance(child, pytree.Leaf) and child.value == ':' - last_was_comma = False - last_was_colon = False - for child in node.children: - if pytree_utils.NodeName(child) == 'comp_for': - self._AppendFirstLeafTokenSubtype( - child, format_token.Subtype.DICT_SET_GENERATOR) - else: - if dict_maker: - if last_was_comma: - self._AppendFirstLeafTokenSubtype( - child, format_token.Subtype.DICTIONARY_KEY) - elif last_was_colon: - if pytree_utils.NodeName(child) == 'power': - self._AppendSubtypeRec(child, format_token.Subtype.NONE) - else: - self._AppendFirstLeafTokenSubtype( - child, format_token.Subtype.DICTIONARY_VALUE) - if style.Get('INDENT_DICTIONARY_VALUE'): - _InsertPseudoParentheses(child) - last_was_comma = isinstance(child, pytree.Leaf) and child.value == ',' - last_was_colon = isinstance(child, pytree.Leaf) and child.value == ':' - self.Visit(child) - - def Visit_expr_stmt(self, node): # pylint: disable=invalid-name - # expr_stmt ::= testlist_star_expr (augassign (yield_expr|testlist) - # | ('=' (yield_expr|testlist_star_expr))*) - for child in node.children: - self.Visit(child) - if isinstance(child, pytree.Leaf) and child.value == '=': - self._AppendTokenSubtype(child, format_token.Subtype.ASSIGN_OPERATOR) - - def Visit_or_test(self, node): # pylint: disable=invalid-name - # or_test ::= and_test ('or' and_test)* - for child in node.children: - self.Visit(child) - if isinstance(child, pytree.Leaf) and child.value == 'or': - self._AppendTokenSubtype(child, format_token.Subtype.BINARY_OPERATOR) - - def Visit_and_test(self, node): # pylint: disable=invalid-name - # and_test ::= not_test ('and' not_test)* - for child in node.children: - self.Visit(child) - if isinstance(child, pytree.Leaf) and child.value == 'and': - self._AppendTokenSubtype(child, format_token.Subtype.BINARY_OPERATOR) - - def Visit_not_test(self, node): # pylint: disable=invalid-name - # not_test ::= 'not' not_test | comparison - for child in node.children: - self.Visit(child) - if isinstance(child, pytree.Leaf) and child.value == 'not': - self._AppendTokenSubtype(child, format_token.Subtype.UNARY_OPERATOR) - - def Visit_comparison(self, node): # pylint: disable=invalid-name - # comparison ::= expr (comp_op expr)* - # comp_op ::= '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not in'|'is'|'is not' - for child in node.children: - self.Visit(child) - if (isinstance(child, pytree.Leaf) and child.value in { - '<', '>', '==', '>=', '<=', '<>', '!=', 'in', 'not in', 'is', 'is not' - }): - self._AppendTokenSubtype(child, format_token.Subtype.BINARY_OPERATOR) - - def Visit_star_expr(self, node): # pylint: disable=invalid-name - # star_expr ::= '*' expr - for child in node.children: - self.Visit(child) - if isinstance(child, pytree.Leaf) and child.value == '*': - self._AppendTokenSubtype(child, format_token.Subtype.UNARY_OPERATOR) - - def Visit_expr(self, node): # pylint: disable=invalid-name - # expr ::= xor_expr ('|' xor_expr)* - for child in node.children: - self.Visit(child) - if isinstance(child, pytree.Leaf) and child.value == '|': - self._AppendTokenSubtype(child, format_token.Subtype.BINARY_OPERATOR) - - def Visit_xor_expr(self, node): # pylint: disable=invalid-name - # xor_expr ::= and_expr ('^' and_expr)* - for child in node.children: - self.Visit(child) - if isinstance(child, pytree.Leaf) and child.value == '^': - self._AppendTokenSubtype(child, format_token.Subtype.BINARY_OPERATOR) - - def Visit_and_expr(self, node): # pylint: disable=invalid-name - # and_expr ::= shift_expr ('&' shift_expr)* - for child in node.children: - self.Visit(child) - if isinstance(child, pytree.Leaf) and child.value == '&': - self._AppendTokenSubtype(child, format_token.Subtype.BINARY_OPERATOR) - - def Visit_shift_expr(self, node): # pylint: disable=invalid-name - # shift_expr ::= arith_expr (('<<'|'>>') arith_expr)* - for child in node.children: - self.Visit(child) - if isinstance(child, pytree.Leaf) and child.value in {'<<', '>>'}: - self._AppendTokenSubtype(child, format_token.Subtype.BINARY_OPERATOR) - - def Visit_arith_expr(self, node): # pylint: disable=invalid-name - # arith_expr ::= term (('+'|'-') term)* - for child in node.children: - self.Visit(child) - if isinstance(child, pytree.Leaf) and child.value in '+-': - self._AppendTokenSubtype(child, format_token.Subtype.BINARY_OPERATOR) - - def Visit_term(self, node): # pylint: disable=invalid-name - # term ::= factor (('*'|'/'|'%'|'//') factor)* - for child in node.children: - self.Visit(child) - if (isinstance(child, pytree.Leaf) and - child.value in {'*', '/', '%', '//'}): - self._AppendTokenSubtype(child, format_token.Subtype.BINARY_OPERATOR) - - def Visit_factor(self, node): # pylint: disable=invalid-name - # factor ::= ('+'|'-'|'~') factor | power - for child in node.children: - self.Visit(child) - if isinstance(child, pytree.Leaf) and child.value in '+-~': - self._AppendTokenSubtype(child, format_token.Subtype.UNARY_OPERATOR) - - def Visit_power(self, node): # pylint: disable=invalid-name - # power ::= atom trailer* ['**' factor] - for child in node.children: - self.Visit(child) - if isinstance(child, pytree.Leaf) and child.value == '**': - self._AppendTokenSubtype(child, format_token.Subtype.BINARY_OPERATOR) - - def Visit_trailer(self, node): # pylint: disable=invalid-name - for child in node.children: - self.Visit(child) - if isinstance(child, pytree.Leaf) and child.value in '[]': - self._AppendTokenSubtype(child, format_token.Subtype.SUBSCRIPT_BRACKET) - - def Visit_subscript(self, node): # pylint: disable=invalid-name - # subscript ::= test | [test] ':' [test] [sliceop] - for child in node.children: - self.Visit(child) - if isinstance(child, pytree.Leaf) and child.value == ':': - self._AppendTokenSubtype(child, format_token.Subtype.SUBSCRIPT_COLON) - - def Visit_sliceop(self, node): # pylint: disable=invalid-name - # sliceop ::= ':' [test] - for child in node.children: - self.Visit(child) - if isinstance(child, pytree.Leaf) and child.value == ':': - self._AppendTokenSubtype(child, format_token.Subtype.SUBSCRIPT_COLON) - - def Visit_argument(self, node): # pylint: disable=invalid-name - # argument ::= - # test [comp_for] | test '=' test - self._ProcessArgLists(node) - - def Visit_arglist(self, node): # pylint: disable=invalid-name - # arglist ::= - # (argument ',')* (argument [','] - # | '*' test (',' argument)* [',' '**' test] - # | '**' test) - self._ProcessArgLists(node) - self._SetDefaultOrNamedAssignArgListSubtype(node) - - def Visit_funcdef(self, node): # pylint: disable=invalid-name - # funcdef ::= - # 'def' NAME parameters ['->' test] ':' suite - for child in node.children: - if pytree_utils.NodeName(child) == 'NAME' and child.value != 'def': - self._AppendTokenSubtype(child, format_token.Subtype.FUNC_DEF) - break - for child in node.children: - self.Visit(child) - - def Visit_typedargslist(self, node): # pylint: disable=invalid-name - # typedargslist ::= - # ((tfpdef ['=' test] ',')* - # ('*' [tname] (',' tname ['=' test])* [',' '**' tname] - # | '**' tname) - # | tfpdef ['=' test] (',' tfpdef ['=' test])* [',']) - self._ProcessArgLists(node) - self._SetDefaultOrNamedAssignArgListSubtype(node) - - def Visit_varargslist(self, node): # pylint: disable=invalid-name - # varargslist ::= - # ((vfpdef ['=' test] ',')* - # ('*' [vname] (',' vname ['=' test])* [',' '**' vname] - # | '**' vname) - # | vfpdef ['=' test] (',' vfpdef ['=' test])* [',']) - self._ProcessArgLists(node) - self._SetDefaultOrNamedAssignArgListSubtype(node) - - def Visit_comp_for(self, node): # pylint: disable=invalid-name - # comp_for ::= 'for' exprlist 'in' testlist_safe [comp_iter] - self._AppendSubtypeRec(node, format_token.Subtype.COMP_FOR) - self.DefaultNodeVisit(node) - - def Visit_comp_if(self, node): # pylint: disable=invalid-name - # comp_if ::= 'if' old_test [comp_iter] - self._AppendSubtypeRec(node, format_token.Subtype.COMP_IF) - self.DefaultNodeVisit(node) - - def _ProcessArgLists(self, node): - """Common method for processing argument lists.""" - for child in node.children: - self.Visit(child) - if isinstance(child, pytree.Leaf): - self._AppendTokenSubtype( - child, - subtype=_ARGLIST_TOKEN_TO_SUBTYPE.get(child.value, - format_token.Subtype.NONE), - force=False) - - def _AppendSubtypeRec(self, node, subtype, force=True): - """Append the leafs in the node to the given subtype.""" - if isinstance(node, pytree.Leaf): - self._AppendTokenSubtype(node, subtype, force=force) - return - for child in node.children: - self._AppendSubtypeRec(child, subtype, force=force) - - def _AppendTokenSubtype(self, node, subtype, force=True): - """Append the token's subtype only if it's not already set.""" - pytree_utils.AppendNodeAnnotation(node, pytree_utils.Annotation.SUBTYPE, - subtype) - - def _AppendFirstLeafTokenSubtype(self, node, subtype, force=False): - """Append the first leaf token's subtypes.""" - if isinstance(node, pytree.Leaf): - self._AppendTokenSubtype(node, subtype, force=force) - return - self._AppendFirstLeafTokenSubtype(node.children[0], subtype, force=force) - - def _SetDefaultOrNamedAssignArgListSubtype(self, node): - - def HasDefaultOrNamedAssignSubtype(node): - if isinstance(node, pytree.Leaf): - if (format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN in - pytree_utils.GetNodeAnnotation( - node, pytree_utils.Annotation.SUBTYPE, set())): - return True - return False - has_subtype = False - for child in node.children: - has_subtype |= HasDefaultOrNamedAssignSubtype(child) - return has_subtype - - if HasDefaultOrNamedAssignSubtype(node): - for child in node.children: - if pytree_utils.NodeName(child) != 'COMMA': - self._AppendFirstLeafTokenSubtype( - child, format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST) - - -def _InsertPseudoParentheses(node): - comment_node = None - if isinstance(node, pytree.Node): - if node.children[-1].type == token.COMMENT: - comment_node = node.children[-1].clone() - node.children[-1].remove() - - first = _GetFirstLeafNode(node) - last = _GetLastLeafNode(node) - - lparen = pytree.Leaf( - token.LPAR, u'(', context=('', (first.get_lineno(), first.column - 1))) - last_lineno = last.get_lineno() - if last.type == token.STRING and '\n' in last.value: - last_lineno += last.value.count('\n') - - if last.type == token.STRING and '\n' in last.value: - last_column = len(last.value.split('\n')[-1]) + 1 - else: - last_column = last.column + len(last.value) + 1 - rparen = pytree.Leaf( - token.RPAR, u')', context=('', (last_lineno, last_column))) - - lparen.is_pseudo = True - rparen.is_pseudo = True - - if isinstance(node, pytree.Node): - node.insert_child(0, lparen) - node.append_child(rparen) - if comment_node: - node.append_child(comment_node) - else: - new_node = pytree.Node(syms.atom, [lparen, node.clone(), rparen]) - node.replace(new_node) - - -def _GetFirstLeafNode(node): - if isinstance(node, pytree.Leaf): - return node - return _GetFirstLeafNode(node.children[0]) - - -def _GetLastLeafNode(node): - if isinstance(node, pytree.Leaf): - return node - return _GetLastLeafNode(node.children[-1]) diff --git a/yapf/yapflib/subtypes.py b/yapf/yapflib/subtypes.py new file mode 100644 index 000000000..3c234fbfb --- /dev/null +++ b/yapf/yapflib/subtypes.py @@ -0,0 +1,41 @@ +# Copyright 2021 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Token subtypes used to improve formatting.""" + +NONE = 0 +UNARY_OPERATOR = 1 +BINARY_OPERATOR = 2 +SUBSCRIPT_COLON = 3 +SUBSCRIPT_BRACKET = 4 +DEFAULT_OR_NAMED_ASSIGN = 5 +DEFAULT_OR_NAMED_ASSIGN_ARG_LIST = 6 +VARARGS_LIST = 7 +VARARGS_STAR = 8 +KWARGS_STAR_STAR = 9 +ASSIGN_OPERATOR = 10 +DICTIONARY_KEY = 11 +DICTIONARY_KEY_PART = 12 +DICTIONARY_VALUE = 13 +DICT_SET_GENERATOR = 14 +COMP_EXPR = 15 +COMP_FOR = 16 +COMP_IF = 17 +FUNC_DEF = 18 +DECORATOR = 19 +TYPED_NAME = 20 +TYPED_NAME_ARG_LIST = 21 +SIMPLE_EXPRESSION = 22 +PARAMETER_START = 23 +PARAMETER_STOP = 24 +LAMBDEF = 25 diff --git a/yapf/yapflib/verifier.py b/yapf/yapflib/verifier.py deleted file mode 100644 index a670ea9db..000000000 --- a/yapf/yapflib/verifier.py +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright 2015-2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Verify that the generated code is valid code. - -This takes a line of code and "normalizes" it. I.e., it transforms the snippet -into something that has the potential to compile. - - VerifyCode(): the main function exported by this module. -""" - -import ast -import re -import sys -import textwrap - - -class InternalError(Exception): - """Internal error in verifying formatted code.""" - pass - - -def VerifyCode(code): - """Verify that the reformatted code is syntactically correct. - - Arguments: - code: (unicode) The reformatted code snippet. - - Raises: - SyntaxError if the code was reformatted incorrectly. - """ - try: - compile(textwrap.dedent(code).encode('UTF-8'), '', 'exec') - except SyntaxError: - try: - ast.parse(textwrap.dedent(code.lstrip('\n')).lstrip(), '', 'exec') - except SyntaxError: - try: - normalized_code = _NormalizeCode(code) - compile(normalized_code.encode('UTF-8'), '', 'exec') - except SyntaxError: - raise InternalError(sys.exc_info()[1]) - - -def _NormalizeCode(code): - """Make sure that the code snippet is compilable.""" - code = textwrap.dedent(code.lstrip('\n')).lstrip() - - # Split the code to lines and get rid of all leading full-comment lines as - # they can mess up the normalization attempt. - lines = code.split('\n') - i = 0 - for i, line in enumerate(lines): - line = line.strip() - if line and not line.startswith('#'): - break - code = '\n'.join(lines[i:]) + '\n' - - if re.match(r'(if|while|for|with|def|class)\b', code): - code += '\n pass' - elif re.match(r'(elif|else)\b', code): - try: - try_code = 'if True:\n pass\n' + code + '\n pass' - ast.parse( - textwrap.dedent(try_code.lstrip('\n')).lstrip(), '', 'exec') - code = try_code - except SyntaxError: - # The assumption here is that the code is on a single line. - code = 'if True: pass\n' + code - elif code.startswith('@'): - code += '\ndef _():\n pass' - elif re.match(r'try\b', code): - code += '\n pass\nexcept:\n pass' - elif re.match(r'(except|finally)\b', code): - code = 'try:\n pass\n' + code + '\n pass' - elif re.match(r'(return|yield)\b', code): - code = 'def _():\n ' + code - elif re.match(r'(continue|break)\b', code): - code = 'while True:\n ' + code - elif re.match(r'print\b', code): - code = 'from __future__ import print_function\n' + code - - return code + '\n' diff --git a/yapf/yapflib/yapf_api.py b/yapf/yapflib/yapf_api.py index f509bbf37..aa010bb8f 100644 --- a/yapf/yapflib/yapf_api.py +++ b/yapf/yapflib/yapf_api.py @@ -1,4 +1,4 @@ -# Copyright 2015-2016 Google Inc. All Rights Reserved. +# Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -29,135 +29,190 @@ than a whole file. print_diff: (bool) Instead of returning the reformatted source, return a diff that turns the formatted source into reformatter source. - verify: (bool) True if reformatted code should be verified for syntax. """ +import codecs import difflib import re -import sys -from lib2to3.pgen2 import tokenize - -from yapf.yapflib import blank_line_calculator -from yapf.yapflib import comment_splicer -from yapf.yapflib import continuation_splicer +from yapf.pyparser import pyparser +from yapf.pytree import blank_line_calculator +from yapf.pytree import comment_splicer +from yapf.pytree import continuation_splicer +from yapf.pytree import pytree_unwrapper +from yapf.pytree import pytree_utils +from yapf.pytree import split_penalty +from yapf.pytree import subtype_assigner +from yapf.yapflib import errors from yapf.yapflib import file_resources -from yapf.yapflib import py3compat -from yapf.yapflib import pytree_unwrapper -from yapf.yapflib import pytree_utils +from yapf.yapflib import identify_container from yapf.yapflib import reformatter -from yapf.yapflib import split_penalty from yapf.yapflib import style -from yapf.yapflib import subtype_assigner def FormatFile(filename, style_config=None, lines=None, print_diff=False, - verify=True, in_place=False, logger=None): """Format a single Python file and return the formatted code. Arguments: filename: (unicode) The file to reformat. + style_config: (string) Either a style name or a path to a file that contains + formatting style settings. If None is specified, use the default style + as set in style.DEFAULT_STYLE_FACTORY + lines: (list of tuples of integers) A list of tuples of lines, [start, end], + that we want to format. The lines are 1-based indexed. It can be used by + third-party code (e.g., IDEs) when reformatting a snippet of code rather + than a whole file. + print_diff: (bool) Instead of returning the reformatted source, return a + diff that turns the formatted source into reformatter source. in_place: (bool) If True, write the reformatted code back to the file. logger: (io streamer) A stream to output logging. - remaining arguments: see comment at the top of this module. Returns: Tuple of (reformatted_code, encoding, changed). reformatted_code is None if - the file is sucessfully written to (having used in_place). reformatted_code + the file is successfully written to (having used in_place). reformatted_code is a diff if print_diff is True. Raises: IOError: raised if there was an error reading the file. ValueError: raised if in_place and print_diff are both specified. """ - _CheckPythonVersion() - if in_place and print_diff: raise ValueError('Cannot pass both in_place and print_diff.') - original_source, encoding = ReadFile(filename, logger) + original_source, newline, encoding = ReadFile(filename, logger) reformatted_source, changed = FormatCode( original_source, style_config=style_config, filename=filename, lines=lines, - print_diff=print_diff, - verify=verify) + print_diff=print_diff) + if newline != '\n': + reformatted_source = reformatted_source.replace('\n', newline) if in_place: - if original_source and original_source != reformatted_source: + if changed: file_resources.WriteReformattedCode(filename, reformatted_source, - in_place, encoding) + encoding, in_place) return None, encoding, changed return reformatted_source, encoding, changed -def FormatCode(unformatted_source, - filename='', - style_config=None, - lines=None, - print_diff=False, - verify=True): - """Format a string of Python code. +def FormatTree(tree, style_config=None, lines=None): + """Format a parsed lib2to3 pytree. This provides an alternative entry point to YAPF. Arguments: - unformatted_source: (unicode) The code to format. - filename: (unicode) The name of the file being reformatted. - remaining arguments: see comment at the top of this module. + tree: (pytree.Node) The root of the pytree to format. + style_config: (string) Either a style name or a path to a file that contains + formatting style settings. If None is specified, use the default style + as set in style.DEFAULT_STYLE_FACTORY + lines: (list of tuples of integers) A list of tuples of lines, [start, end], + that we want to format. The lines are 1-based indexed. It can be used by + third-party code (e.g., IDEs) when reformatting a snippet of code rather + than a whole file. Returns: - Tuple of (reformatted_source, changed). reformatted_source conforms to the - desired formatting style. changed is True if the source changed. + The source formatted according to the given formatting style. """ - _CheckPythonVersion() style.SetGlobalStyle(style.CreateStyleFromConfig(style_config)) - if not unformatted_source.endswith('\n'): - unformatted_source += '\n' - tree = pytree_utils.ParseCodeToTree(unformatted_source) # Run passes on the tree, modifying it in place. comment_splicer.SpliceComments(tree) continuation_splicer.SpliceContinuations(tree) subtype_assigner.AssignSubtypes(tree) + identify_container.IdentifyContainers(tree) split_penalty.ComputeSplitPenalties(tree) blank_line_calculator.CalculateBlankLines(tree) - uwlines = pytree_unwrapper.UnwrapPyTree(tree) - for uwl in uwlines: - uwl.CalculateFormattingInformation() + llines = pytree_unwrapper.UnwrapPyTree(tree) + for lline in llines: + lline.CalculateFormattingInformation() - _MarkLinesToFormat(uwlines, lines) - reformatted_source = reformatter.Reformat(uwlines, verify) + lines = _LineRangesToSet(lines) + _MarkLinesToFormat(llines, lines) + return reformatter.Reformat(_SplitSemicolons(llines), lines) + + +def FormatAST(ast, style_config=None, lines=None): + """Format a parsed lib2to3 pytree. + + This provides an alternative entry point to YAPF. + + Arguments: + unformatted_source: (unicode) The code to format. + style_config: (string) Either a style name or a path to a file that contains + formatting style settings. If None is specified, use the default style + as set in style.DEFAULT_STYLE_FACTORY + lines: (list of tuples of integers) A list of tuples of lines, [start, end], + that we want to format. The lines are 1-based indexed. It can be used by + third-party code (e.g., IDEs) when reformatting a snippet of code rather + than a whole file. + + Returns: + The source formatted according to the given formatting style. + """ + style.SetGlobalStyle(style.CreateStyleFromConfig(style_config)) + + llines = pyparser.ParseCode(ast) + for lline in llines: + lline.CalculateFormattingInformation() + + lines = _LineRangesToSet(lines) + _MarkLinesToFormat(llines, lines) + return reformatter.Reformat(_SplitSemicolons(llines), lines) + + +def FormatCode(unformatted_source, + filename='', + style_config=None, + lines=None, + print_diff=False): + """Format a string of Python code. + + This provides an alternative entry point to YAPF. + + Arguments: + unformatted_source: (unicode) The code to format. + filename: (unicode) The name of the file being reformatted. + style_config: (string) Either a style name or a path to a file that contains + formatting style settings. If None is specified, use the default style + as set in style.DEFAULT_STYLE_FACTORY + lines: (list of tuples of integers) A list of tuples of lines, [start, end], + that we want to format. The lines are 1-based indexed. It can be used by + third-party code (e.g., IDEs) when reformatting a snippet of code rather + than a whole file. + print_diff: (bool) Instead of returning the reformatted source, return a + diff that turns the formatted source into reformatter source. + + Returns: + Tuple of (reformatted_source, changed). reformatted_source conforms to the + desired formatting style. changed is True if the source changed. + """ + try: + tree = pytree_utils.ParseCodeToTree(unformatted_source) + except Exception as e: + e.filename = filename + raise errors.YapfError(errors.FormatErrorMsg(e)) + + reformatted_source = FormatTree(tree, style_config=style_config, lines=lines) if unformatted_source == reformatted_source: return '' if print_diff else reformatted_source, False - code_diff = _GetUnifiedDiff( - unformatted_source, reformatted_source, filename=filename) - if print_diff: - return code_diff, code_diff != '' + code_diff = _GetUnifiedDiff( + unformatted_source, reformatted_source, filename=filename) + return code_diff, code_diff.strip() != '' # pylint: disable=g-explicit-bool-comparison # noqa return reformatted_source, True -def _CheckPythonVersion(): # pragma: no cover - errmsg = 'yapf is only supported for Python 2.7 or 3.4+' - if sys.version_info[0] == 2: - if sys.version_info[1] < 7: - raise RuntimeError(errmsg) - elif sys.version_info[0] == 3: - if sys.version_info[1] < 4: - raise RuntimeError(errmsg) - - def ReadFile(filename, logger=None): """Read the contents of the file. @@ -176,53 +231,75 @@ def ReadFile(filename, logger=None): IOError: raised if there was an error reading the file. """ try: - with open(filename, 'rb') as fd: - encoding = tokenize.detect_encoding(fd.readline)[0] - except IOError as err: + encoding = file_resources.FileEncoding(filename) + + # Preserves line endings. + with codecs.open(filename, mode='r', encoding=encoding) as fd: + lines = fd.readlines() + + line_ending = file_resources.LineEnding(lines) + source = '\n'.join(line.rstrip('\r\n') for line in lines) + '\n' + return source, line_ending, encoding + except IOError as e: # pragma: no cover if logger: - logger(err) + logger(e) + e.args = (e.args[0], (filename, e.args[1][1], e.args[1][2], e.args[1][3])) raise - - try: - with py3compat.open_with_encoding( - filename, mode='r', encoding=encoding) as fd: - source = fd.read() - return source, encoding - except IOError as err: # pragma: no cover + except UnicodeDecodeError as e: # pragma: no cover if logger: - logger(err) + logger('Could not parse %s! Consider excluding this file with --exclude.', + filename) + logger(e) + e.args = (e.args[0], (filename, e.args[1][1], e.args[1][2], e.args[1][3])) raise -DISABLE_PATTERN = r'^#+ +yapf: *disable$' -ENABLE_PATTERN = r'^#+ +yapf: *enable$' +def _SplitSemicolons(lines): + res = [] + for line in lines: + res.extend(line.Split()) + return res + + +DISABLE_PATTERN = r'^#.*\b(?:yapf:\s*disable|fmt: ?off)\b' +ENABLE_PATTERN = r'^#.*\b(?:yapf:\s*enable|fmt: ?on)\b' -def _MarkLinesToFormat(uwlines, lines): +def _LineRangesToSet(line_ranges): + """Return a set of lines in the range.""" + + if line_ranges is None: + return None + + line_set = set() + for low, high in sorted(line_ranges): + line_set.update(range(low, high + 1)) + + return line_set + + +def _MarkLinesToFormat(llines, lines): """Skip sections of code that we shouldn't reformat.""" if lines: - for uwline in uwlines: - uwline.disable = True - - for start, end in sorted(lines): - for uwline in uwlines: - if uwline.lineno > end: - break - if uwline.lineno >= start: - uwline.disable = False - elif uwline.last.lineno >= start: - uwline.disable = False + for uwline in llines: + uwline.disable = not lines.intersection( + range(uwline.lineno, uwline.last.lineno + 1)) + # Now go through the lines and disable any lines explicitly marked as + # disabled. index = 0 - while index < len(uwlines): - uwline = uwlines[index] + while index < len(llines): + uwline = llines[index] if uwline.is_comment: if _DisableYAPF(uwline.first.value.strip()): - while index < len(uwlines): - uwline = uwlines[index] + index += 1 + while index < len(llines): + uwline = llines[index] + line = uwline.first.value.strip() + if uwline.is_comment and _EnableYAPF(line): + if not _DisableYAPF(line): + break uwline.disable = True - if uwline.is_comment and _EnableYAPF(uwline.first.value.strip()): - break index += 1 elif re.search(DISABLE_PATTERN, uwline.last.value.strip(), re.IGNORECASE): uwline.disable = True @@ -230,17 +307,17 @@ def _MarkLinesToFormat(uwlines, lines): def _DisableYAPF(line): - return (re.search(DISABLE_PATTERN, line.split('\n')[0].strip(), - re.IGNORECASE) or re.search(DISABLE_PATTERN, - line.split('\n')[-1].strip(), - re.IGNORECASE)) + return (re.search(DISABLE_PATTERN, + line.split('\n')[0].strip(), re.IGNORECASE) or + re.search(DISABLE_PATTERN, + line.split('\n')[-1].strip(), re.IGNORECASE)) def _EnableYAPF(line): - return (re.search(ENABLE_PATTERN, line.split('\n')[0].strip(), - re.IGNORECASE) or re.search(ENABLE_PATTERN, - line.split('\n')[-1].strip(), - re.IGNORECASE)) + return (re.search(ENABLE_PATTERN, + line.split('\n')[0].strip(), re.IGNORECASE) or + re.search(ENABLE_PATTERN, + line.split('\n')[-1].strip(), re.IGNORECASE)) def _GetUnifiedDiff(before, after, filename='code'): diff --git a/yapftests/__init__.py b/yapftests/__init__.py index 8b1378917..e7522b2ca 100644 --- a/yapftests/__init__.py +++ b/yapftests/__init__.py @@ -1 +1,13 @@ - +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/yapftests/blank_line_calculator_test.py b/yapftests/blank_line_calculator_test.py index 69d95ce9b..7c9ab0fe9 100644 --- a/yapftests/blank_line_calculator_test.py +++ b/yapftests/blank_line_calculator_test.py @@ -1,4 +1,4 @@ -# Copyright 2015-2016 Google Inc. All Rights Reserved. +# Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,51 +13,21 @@ # limitations under the License. """Tests for yapf.blank_line_calculator.""" -import difflib -import sys import textwrap import unittest -from yapf.yapflib import blank_line_calculator -from yapf.yapflib import comment_splicer -from yapf.yapflib import pytree_unwrapper -from yapf.yapflib import pytree_utils -from yapf.yapflib import pytree_visitor from yapf.yapflib import reformatter -from yapf.yapflib import split_penalty from yapf.yapflib import style -from yapf.yapflib import subtype_assigner - - -class BlankLineCalculatorTest(unittest.TestCase): - - def assertCodeEqual(self, expected_code, code): - if code != expected_code: - msg = ['Code format mismatch:', 'Expected:'] - linelen = style.Get('COLUMN_LIMIT') - for l in expected_code.splitlines(): - if len(l) > linelen: - msg.append('!> %s' % l) - else: - msg.append(' > %s' % l) - msg.append('Actual:') - for l in code.splitlines(): - if len(l) > linelen: - msg.append('!> %s' % l) - else: - msg.append(' > %s' % l) - msg.append('Diff:') - msg.extend( - difflib.unified_diff( - code.splitlines(), - expected_code.splitlines(), - fromfile='actual', - tofile='expected', - lineterm='')) - self.fail('\n'.join(msg)) - - -class BasicBlankLineCalculatorTest(BlankLineCalculatorTest): +from yapf.yapflib import yapf_api + +from yapftests import yapf_test_helper + + +class BasicBlankLineCalculatorTest(yapf_test_helper.YAPFTest): + + @classmethod + def setUpClass(cls): + style.SetGlobalStyle(style.CreateYapfStyle()) def testDecorators(self): unformatted_code = textwrap.dedent("""\ @@ -65,14 +35,14 @@ def testDecorators(self): def foo(): pass - """) + """) expected_formatted_code = textwrap.dedent("""\ @bork() def foo(): pass - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) def testComplexDecorators(self): unformatted_code = textwrap.dedent("""\ @@ -89,7 +59,7 @@ class moo(object): def method(self): pass - """) + """) expected_formatted_code = textwrap.dedent("""\ import sys @@ -106,9 +76,9 @@ class moo(object): @baz() def method(self): pass - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) def testCodeAfterFunctionsAndClasses(self): unformatted_code = textwrap.dedent("""\ @@ -126,7 +96,7 @@ def method_2(self): raise Error except Error as error: pass - """) + """) expected_formatted_code = textwrap.dedent("""\ def foo(): pass @@ -151,9 +121,9 @@ def method_2(self): raise Error except Error as error: pass - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) def testCommentSpacing(self): unformatted_code = textwrap.dedent("""\ @@ -184,7 +154,7 @@ def foo(self): # Another multiline # comment pass - """) + """) expected_formatted_code = textwrap.dedent("""\ # This is the first comment # And it's multiline @@ -195,6 +165,7 @@ def foo(self): def foo(): pass + # multiline before a # class definition @@ -216,9 +187,9 @@ def foo(self): # Another multiline # comment pass - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) def testCommentBeforeMethod(self): code = textwrap.dedent("""\ @@ -227,9 +198,9 @@ class foo(object): # pylint: disable=invalid-name def f(self): pass - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) def testCommentsBeforeClassDefs(self): code = textwrap.dedent('''\ @@ -240,19 +211,19 @@ def testCommentsBeforeClassDefs(self): class Foo(object): pass - ''') - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + ''') + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) - def testComemntsBeforeDecorator(self): + def testCommentsBeforeDecorator(self): code = textwrap.dedent("""\ # The @foo operator adds bork to a(). @foo() def a(): pass - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) code = textwrap.dedent("""\ # Hello world @@ -261,9 +232,26 @@ def a(): @foo() def a(): pass - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testCommentsAfterDecorator(self): + code = textwrap.dedent("""\ + class _(): + + def _(): + pass + + @pytest.mark.xfail(reason="#709 and #710") + # also + #@pytest.mark.xfail(setuptools.tests.is_ascii, + # reason="https://github.com/pypa/setuptools/issues/706") + def test_unicode_filename_in_sdist(self, sdist_unicode, tmpdir, monkeypatch): + pass + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) def testInnerClasses(self): unformatted_code = textwrap.dedent("""\ @@ -273,7 +261,7 @@ class Error(Exception): pass class TaskValidationError(Error): pass class DeployAPIHTTPError(Error): pass - """) + """) expected_formatted_code = textwrap.dedent("""\ class DeployAPIClient(object): @@ -285,40 +273,149 @@ class TaskValidationError(Error): class DeployAPIHTTPError(Error): pass - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testLinesOnRangeBoundary(self): + unformatted_code = textwrap.dedent("""\ + def A(): + pass + + def B(): # 4 + pass # 5 + + def C(): + pass + def D(): # 9 + pass # 10 + def E(): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + def A(): + pass + + + def B(): # 4 + pass # 5 + + def C(): + pass + + + def D(): # 9 + pass # 10 + def E(): + pass + """) + code, changed = yapf_api.FormatCode( + unformatted_code, lines=[(4, 5), (9, 10)]) + self.assertCodeEqual(expected_formatted_code, code) + self.assertTrue(changed) + + def testLinesRangeBoundaryNotOutside(self): + unformatted_code = textwrap.dedent("""\ + def A(): + pass + + + + def B(): # 6 + pass # 7 + + + + def C(): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + def A(): + pass + + + + def B(): # 6 + pass # 7 + + + + def C(): + pass + """) + code, changed = yapf_api.FormatCode(unformatted_code, lines=[(6, 7)]) + self.assertCodeEqual(expected_formatted_code, code) + self.assertFalse(changed) + + def testLinesRangeRemove(self): + unformatted_code = textwrap.dedent("""\ + def A(): + pass + + + + def B(): # 6 + pass # 7 + + + + + def C(): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + def A(): + pass + + + def B(): # 6 + pass # 7 + + + + + def C(): + pass + """) + code, changed = yapf_api.FormatCode(unformatted_code, lines=[(5, 9)]) + self.assertCodeEqual(expected_formatted_code, code) + self.assertTrue(changed) + + def testLinesRangeRemoveSome(self): + unformatted_code = textwrap.dedent("""\ + def A(): + pass + -def _ParseAndUnwrap(code, dumptree=False): - """Produces unwrapped lines from the given code. - Parses the code into a tree, performs comment splicing and runs the - unwrapper. + def B(): # 7 + pass # 8 - Arguments: - code: code to parse as a string - dumptree: if True, the parsed pytree (after comment splicing) is dumped - to stderr. Useful for debugging. - Returns: - List of unwrapped lines. - """ - style.SetGlobalStyle(style.CreateChromiumStyle()) - tree = pytree_utils.ParseCodeToTree(code) - comment_splicer.SpliceComments(tree) - subtype_assigner.AssignSubtypes(tree) - split_penalty.ComputeSplitPenalties(tree) - blank_line_calculator.CalculateBlankLines(tree) - if dumptree: - pytree_visitor.DumpPyTree(tree, target_stream=sys.stderr) - uwlines = pytree_unwrapper.UnwrapPyTree(tree) - for uwl in uwlines: - uwl.CalculateFormattingInformation() + def C(): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + def A(): + pass + + - return uwlines + def B(): # 7 + pass # 8 + + + + + def C(): + pass + """) + code, changed = yapf_api.FormatCode(unformatted_code, lines=[(6, 9)]) + self.assertCodeEqual(expected_formatted_code, code) + self.assertTrue(changed) if __name__ == '__main__': diff --git a/yapftests/comment_splicer_test.py b/yapftests/comment_splicer_test.py index c26cac8c1..3a63da7ca 100644 --- a/yapftests/comment_splicer_test.py +++ b/yapftests/comment_splicer_test.py @@ -1,4 +1,4 @@ -# Copyright 2015-2016 Google Inc. All Rights Reserved. +# Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,12 +16,13 @@ import textwrap import unittest -from yapf.yapflib import comment_splicer -from yapf.yapflib import py3compat -from yapf.yapflib import pytree_utils +from yapf.pytree import comment_splicer +from yapf.pytree import pytree_utils +from yapftests import yapf_test_helper -class CommentSplicerTest(unittest.TestCase): + +class CommentSplicerTest(yapf_test_helper.YAPFTest): def _AssertNodeType(self, expected_type, node): self.assertEqual(expected_type, pytree_utils.NodeName(node)) @@ -38,14 +39,15 @@ def _AssertNodeIsComment(self, node, text_in_comment=None): def _FindNthChildNamed(self, node, name, n=1): for i, child in enumerate( - py3compat.ifilter(lambda c: pytree_utils.NodeName(c) == name, - node.pre_order())): + [c for c in node.pre_order() if pytree_utils.NodeName(c) == name]): if i == n - 1: return child raise RuntimeError('No Nth child for n={0}'.format(n)) def testSimpleInline(self): - code = 'foo = 1 # and a comment\n' + code = textwrap.dedent("""\ + foo = 1 # and a comment + """) tree = pytree_utils.ParseCodeToTree(code) comment_splicer.SpliceComments(tree) @@ -58,11 +60,11 @@ def testSimpleInline(self): self._AssertNodeIsComment(comment_node, '# and a comment') def testSimpleSeparateLine(self): - code = textwrap.dedent(r''' - foo = 1 - # first comment - bar = 2 - ''') + code = textwrap.dedent("""\ + foo = 1 + # first comment + bar = 2 + """) tree = pytree_utils.ParseCodeToTree(code) comment_splicer.SpliceComments(tree) @@ -73,12 +75,12 @@ def testSimpleSeparateLine(self): self._AssertNodeIsComment(comment_node) def testTwoLineComment(self): - code = textwrap.dedent(r''' - foo = 1 - # first comment - # second comment - bar = 2 - ''') + code = textwrap.dedent("""\ + foo = 1 + # first comment + # second comment + bar = 2 + """) tree = pytree_utils.ParseCodeToTree(code) comment_splicer.SpliceComments(tree) @@ -87,11 +89,11 @@ def testTwoLineComment(self): self._AssertNodeIsComment(tree.children[1]) def testCommentIsFirstChildInCompound(self): - code = textwrap.dedent(r''' - if x: - # a comment - foo = 1 - ''') + code = textwrap.dedent(""" + if x: + # a comment + foo = 1 + """) tree = pytree_utils.ParseCodeToTree(code) comment_splicer.SpliceComments(tree) @@ -103,11 +105,11 @@ def testCommentIsFirstChildInCompound(self): self._AssertNodeIsComment(if_suite.children[1]) def testCommentIsLastChildInCompound(self): - code = textwrap.dedent(r''' - if x: - foo = 1 - # a comment - ''') + code = textwrap.dedent("""\ + if x: + foo = 1 + # a comment + """) tree = pytree_utils.ParseCodeToTree(code) comment_splicer.SpliceComments(tree) @@ -119,11 +121,11 @@ def testCommentIsLastChildInCompound(self): self._AssertNodeIsComment(if_suite.children[-2]) def testInlineAfterSeparateLine(self): - code = textwrap.dedent(r''' - bar = 1 - # line comment - foo = 1 # inline comment - ''') + code = textwrap.dedent("""\ + bar = 1 + # line comment + foo = 1 # inline comment + """) tree = pytree_utils.ParseCodeToTree(code) comment_splicer.SpliceComments(tree) @@ -137,11 +139,11 @@ def testInlineAfterSeparateLine(self): self._AssertNodeIsComment(inline_comment_node, '# inline comment') def testSeparateLineAfterInline(self): - code = textwrap.dedent(r''' - bar = 1 - foo = 1 # inline comment - # line comment - ''') + code = textwrap.dedent("""\ + bar = 1 + foo = 1 # inline comment + # line comment + """) tree = pytree_utils.ParseCodeToTree(code) comment_splicer.SpliceComments(tree) @@ -155,12 +157,12 @@ def testSeparateLineAfterInline(self): self._AssertNodeIsComment(inline_comment_node, '# inline comment') def testCommentBeforeDedent(self): - code = textwrap.dedent(r''' - if bar: - z = 1 - # a comment - j = 2 - ''') + code = textwrap.dedent("""\ + if bar: + z = 1 + # a comment + j = 2 + """) tree = pytree_utils.ParseCodeToTree(code) comment_splicer.SpliceComments(tree) @@ -170,13 +172,13 @@ def testCommentBeforeDedent(self): self._AssertNodeType('DEDENT', if_suite.children[-1]) def testCommentBeforeDedentTwoLevel(self): - code = textwrap.dedent(r''' - if foo: - if bar: - z = 1 - # a comment - y = 1 - ''') + code = textwrap.dedent("""\ + if foo: + if bar: + z = 1 + # a comment + y = 1 + """) tree = pytree_utils.ParseCodeToTree(code) comment_splicer.SpliceComments(tree) @@ -187,13 +189,13 @@ def testCommentBeforeDedentTwoLevel(self): self._AssertNodeType('DEDENT', if_suite.children[-1]) def testCommentBeforeDedentTwoLevelImproperlyIndented(self): - code = textwrap.dedent(r''' - if foo: - if bar: - z = 1 - # comment 2 - y = 1 - ''') + code = textwrap.dedent("""\ + if foo: + if bar: + z = 1 + # comment 2 + y = 1 + """) tree = pytree_utils.ParseCodeToTree(code) comment_splicer.SpliceComments(tree) @@ -206,14 +208,41 @@ def testCommentBeforeDedentTwoLevelImproperlyIndented(self): self._AssertNodeIsComment(if_suite.children[-2]) self._AssertNodeType('DEDENT', if_suite.children[-1]) + def testCommentBeforeDedentThreeLevel(self): + code = textwrap.dedent("""\ + if foo: + if bar: + z = 1 + # comment 2 + # comment 1 + # comment 0 + j = 2 + """) + tree = pytree_utils.ParseCodeToTree(code) + comment_splicer.SpliceComments(tree) + + # comment 0 should go under the tree root + self._AssertNodeIsComment(tree.children[1], '# comment 0') + + # comment 1 is in the first if_suite, right before the DEDENT + if_suite_1 = self._FindNthChildNamed(tree, 'suite', n=1) + self._AssertNodeIsComment(if_suite_1.children[-2], '# comment 1') + self._AssertNodeType('DEDENT', if_suite_1.children[-1]) + + # comment 2 is in if_suite nested under the first if suite, + # right before the DEDENT + if_suite_2 = self._FindNthChildNamed(tree, 'suite', n=2) + self._AssertNodeIsComment(if_suite_2.children[-2], '# comment 2') + self._AssertNodeType('DEDENT', if_suite_2.children[-1]) + def testCommentsInClass(self): - code = textwrap.dedent(r''' - class Foo: - """docstring abc...""" - # top-level comment - def foo(): pass - # another comment - ''') + code = textwrap.dedent("""\ + class Foo: + '''docstring abc...''' + # top-level comment + def foo(): pass + # another comment + """) tree = pytree_utils.ParseCodeToTree(code) comment_splicer.SpliceComments(tree) @@ -229,13 +258,13 @@ def foo(): pass self._AssertNodeIsComment(toplevel_comment, '# top-level') def testMultipleBlockComments(self): - code = textwrap.dedent(r''' + code = textwrap.dedent("""\ # Block comment number 1 # Block comment number 2 def f(): pass - ''') + """) tree = pytree_utils.ParseCodeToTree(code) comment_splicer.SpliceComments(tree) @@ -248,7 +277,7 @@ def f(): self._AssertNodeIsComment(block_comment_2, '# Block comment number 2') def testCommentsOnDedents(self): - code = textwrap.dedent(r''' + code = textwrap.dedent("""\ class Foo(object): # A comment for qux. def qux(self): @@ -258,7 +287,7 @@ def qux(self): def mux(self): pass - ''') + """) tree = pytree_utils.ParseCodeToTree(code) comment_splicer.SpliceComments(tree) @@ -272,10 +301,10 @@ def mux(self): self._AssertNodeIsComment(interim_comment, '# Interim comment.') def testExprComments(self): - code = textwrap.dedent(r''' - foo( # Request fractions of an hour. - 948.0/3600, 20) - ''') + code = textwrap.dedent("""\ + foo( # Request fractions of an hour. + 948.0/3600, 20) + """) tree = pytree_utils.ParseCodeToTree(code) comment_splicer.SpliceComments(tree) @@ -284,12 +313,12 @@ def testExprComments(self): self._AssertNodeIsComment(comment, '# Request fractions of an hour.') def testMultipleCommentsInOneExpr(self): - code = textwrap.dedent(r''' - foo( # com 1 - 948.0/3600, # com 2 - 20 + 12 # com 3 - ) - ''') + code = textwrap.dedent("""\ + foo( # com 1 + 948.0/3600, # com 2 + 20 + 12 # com 3 + ) + """) tree = pytree_utils.ParseCodeToTree(code) comment_splicer.SpliceComments(tree) diff --git a/yapftests/file_resources_test.py b/yapftests/file_resources_test.py index 0f2cfd013..e71742c6a 100644 --- a/yapftests/file_resources_test.py +++ b/yapftests/file_resources_test.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2015-2016 Google Inc. All Rights Reserved. +# Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,34 +14,113 @@ # limitations under the License. """Tests for yapf.file_resources.""" +import codecs import contextlib import os import shutil -import sys import tempfile import unittest +from io import BytesIO from yapf.yapflib import errors from yapf.yapflib import file_resources -from yapf.yapflib import py3compat + +from yapftests import utils +from yapftests import yapf_test_helper @contextlib.contextmanager -def stdout_redirector(stream): # pylint: disable=invalid-name - old_stdout = sys.stdout - sys.stdout = stream +def _restore_working_dir(): + curdir = os.getcwd() try: yield finally: - sys.stdout = old_stdout + os.chdir(curdir) -class GetDefaultStyleForDirTest(unittest.TestCase): +@contextlib.contextmanager +def _exists_mocked_in_module(module, mock_implementation): + unmocked_exists = getattr(module, 'exists') + setattr(module, 'exists', mock_implementation) + try: + yield + finally: + setattr(module, 'exists', unmocked_exists) - def setUp(self): + +class GetExcludePatternsForDir(yapf_test_helper.YAPFTest): + + def setUp(self): # pylint: disable=g-missing-super-call self.test_tmpdir = tempfile.mkdtemp() - def tearDown(self): + def tearDown(self): # pylint: disable=g-missing-super-call + shutil.rmtree(self.test_tmpdir) + + def test_get_exclude_file_patterns_from_yapfignore(self): + local_ignore_file = os.path.join(self.test_tmpdir, '.yapfignore') + ignore_patterns = ['temp/**/*.py', 'temp2/*.py'] + with open(local_ignore_file, 'w') as f: + f.writelines('\n'.join(ignore_patterns)) + + self.assertEqual( + sorted(file_resources.GetExcludePatternsForDir(self.test_tmpdir)), + sorted(ignore_patterns)) + + def test_get_exclude_file_patterns_from_yapfignore_with_wrong_syntax(self): + local_ignore_file = os.path.join(self.test_tmpdir, '.yapfignore') + ignore_patterns = ['temp/**/*.py', './wrong/syntax/*.py'] + with open(local_ignore_file, 'w') as f: + f.writelines('\n'.join(ignore_patterns)) + + with self.assertRaises(errors.YapfError): + file_resources.GetExcludePatternsForDir(self.test_tmpdir) + + def test_get_exclude_file_patterns_from_pyproject(self): + local_ignore_file = os.path.join(self.test_tmpdir, 'pyproject.toml') + ignore_patterns = ['temp/**/*.py', 'temp2/*.py'] + with open(local_ignore_file, 'w') as f: + f.write('[tool.yapfignore]\n') + f.write('ignore_patterns=[') + f.writelines('\n,'.join(['"{}"'.format(p) for p in ignore_patterns])) + f.write(']') + + self.assertEqual( + sorted(file_resources.GetExcludePatternsForDir(self.test_tmpdir)), + sorted(ignore_patterns)) + + def test_get_exclude_file_patterns_from_pyproject_no_ignore_section(self): + local_ignore_file = os.path.join(self.test_tmpdir, 'pyproject.toml') + ignore_patterns = [] + open(local_ignore_file, 'w').close() + + self.assertEqual( + sorted(file_resources.GetExcludePatternsForDir(self.test_tmpdir)), + sorted(ignore_patterns)) + + def test_get_exclude_file_patterns_from_pyproject_ignore_section_empty(self): + local_ignore_file = os.path.join(self.test_tmpdir, 'pyproject.toml') + ignore_patterns = [] + with open(local_ignore_file, 'w') as f: + f.write('[tool.yapfignore]\n') + + self.assertEqual( + sorted(file_resources.GetExcludePatternsForDir(self.test_tmpdir)), + sorted(ignore_patterns)) + + def test_get_exclude_file_patterns_with_no_config_files(self): + ignore_patterns = [] + + self.assertEqual( + sorted(file_resources.GetExcludePatternsForDir(self.test_tmpdir)), + sorted(ignore_patterns)) + + +class GetDefaultStyleForDirTest(yapf_test_helper.YAPFTest): + + def setUp(self): # pylint: disable=g-missing-super-call + self.test_tmpdir = tempfile.mkdtemp() + + def tearDown(self): # pylint: disable=g-missing-super-call shutil.rmtree(self.test_tmpdir) def test_no_local_style(self): @@ -49,6 +128,12 @@ def test_no_local_style(self): style_name = file_resources.GetDefaultStyleForDir(test_file) self.assertEqual(style_name, 'pep8') + def test_no_local_style_custom_default(self): + test_file = os.path.join(self.test_tmpdir, 'file.py') + style_name = file_resources.GetDefaultStyleForDir( + test_file, default_style='custom-default') + self.assertEqual(style_name, 'custom-default') + def test_with_local_style(self): # Create an empty .style.yapf file in test_tmpdir style_file = os.path.join(self.test_tmpdir, '.style.yapf') @@ -62,22 +147,75 @@ def test_with_local_style(self): self.assertEqual(style_file, file_resources.GetDefaultStyleForDir(test_filename)) + def test_setup_config(self): + # An empty setup.cfg file should not be used + setup_config = os.path.join(self.test_tmpdir, 'setup.cfg') + open(setup_config, 'w').close() + + test_dir = os.path.join(self.test_tmpdir, 'dir1') + style_name = file_resources.GetDefaultStyleForDir(test_dir) + self.assertEqual(style_name, 'pep8') + + # One with a '[yapf]' section should be used + with open(setup_config, 'w') as f: + f.write('[yapf]\n') + self.assertEqual(setup_config, + file_resources.GetDefaultStyleForDir(test_dir)) + + def test_pyproject_toml(self): + pyproject_toml = os.path.join(self.test_tmpdir, 'pyproject.toml') + open(pyproject_toml, 'w').close() + + test_dir = os.path.join(self.test_tmpdir, 'dir1') + style_name = file_resources.GetDefaultStyleForDir(test_dir) + self.assertEqual(style_name, 'pep8') + + # One with a '[tool.yapf]' section should be used + with open(pyproject_toml, 'w') as f: + f.write('[tool.yapf]\n') + self.assertEqual(pyproject_toml, + file_resources.GetDefaultStyleForDir(test_dir)) + + def test_local_style_at_root(self): + # Test behavior of files located on the root, and under root. + rootdir = os.path.abspath(os.path.sep) + test_dir_at_root = os.path.join(rootdir, 'dir1') + test_dir_under_root = os.path.join(rootdir, 'dir1', 'dir2') + + # Fake placing only a style file at the root by mocking `os.path.exists`. + style_file = os.path.join(rootdir, '.style.yapf') + + def mock_exists_implementation(path): + return path == style_file + + with _exists_mocked_in_module(file_resources.os.path, + mock_exists_implementation): + # Both files should find the style file at the root. + default_style_at_root = file_resources.GetDefaultStyleForDir( + test_dir_at_root) + self.assertEqual(style_file, default_style_at_root) + default_style_under_root = file_resources.GetDefaultStyleForDir( + test_dir_under_root) + self.assertEqual(style_file, default_style_under_root) + def _touch_files(filenames): for name in filenames: open(name, 'a').close() -class GetCommandLineFilesTest(unittest.TestCase): +class GetCommandLineFilesTest(yapf_test_helper.YAPFTest): - def setUp(self): + def setUp(self): # pylint: disable=g-missing-super-call self.test_tmpdir = tempfile.mkdtemp() + self.old_dir = os.getcwd() - def tearDown(self): + def tearDown(self): # pylint: disable=g-missing-super-call + os.chdir(self.old_dir) shutil.rmtree(self.test_tmpdir) def _make_test_dir(self, name): - fullpath = os.path.join(self.test_tmpdir, name) + fullpath = os.path.normpath(os.path.join(self.test_tmpdir, name)) os.makedirs(fullpath) return fullpath @@ -89,11 +227,13 @@ def test_find_files_not_dirs(self): _touch_files([file1, file2]) self.assertEqual( - file_resources.GetCommandLineFiles( - [file1, file2], recursive=False, exclude=None), [file1, file2]) + file_resources.GetCommandLineFiles([file1, file2], + recursive=False, + exclude=None), [file1, file2]) self.assertEqual( - file_resources.GetCommandLineFiles( - [file1, file2], recursive=True, exclude=None), [file1, file2]) + file_resources.GetCommandLineFiles([file1, file2], + recursive=True, + exclude=None), [file1, file2]) def test_nonrecursive_find_in_dir(self): tdir1 = self._make_test_dir('test1') @@ -113,40 +253,150 @@ def test_recursive_find_in_dir(self): tdir1 = self._make_test_dir('test1') tdir2 = self._make_test_dir('test2/testinner/') tdir3 = self._make_test_dir('test3/foo/bar/bas/xxx') - files = [os.path.join(tdir1, 'testfile1.py'), - os.path.join(tdir2, 'testfile2.py'), - os.path.join(tdir3, 'testfile3.py')] + files = [ + os.path.join(tdir1, 'testfile1.py'), + os.path.join(tdir2, 'testfile2.py'), + os.path.join(tdir3, 'testfile3.py'), + ] _touch_files(files) self.assertEqual( sorted( - file_resources.GetCommandLineFiles( - [self.test_tmpdir], recursive=True, exclude=None)), - sorted(files)) + file_resources.GetCommandLineFiles([self.test_tmpdir], + recursive=True, + exclude=None)), sorted(files)) def test_recursive_find_in_dir_with_exclude(self): tdir1 = self._make_test_dir('test1') tdir2 = self._make_test_dir('test2/testinner/') tdir3 = self._make_test_dir('test3/foo/bar/bas/xxx') - files = [os.path.join(tdir1, 'testfile1.py'), - os.path.join(tdir2, 'testfile2.py'), - os.path.join(tdir3, 'testfile3.py')] + files = [ + os.path.join(tdir1, 'testfile1.py'), + os.path.join(tdir2, 'testfile2.py'), + os.path.join(tdir3, 'testfile3.py'), + ] _touch_files(files) self.assertEqual( sorted( - file_resources.GetCommandLineFiles( - [self.test_tmpdir], recursive=True, exclude=['*test*3.py'])), - sorted([os.path.join(tdir1, 'testfile1.py'), - os.path.join(tdir2, 'testfile2.py')])) + file_resources.GetCommandLineFiles([self.test_tmpdir], + recursive=True, + exclude=['*test*3.py'])), + sorted([ + os.path.join(tdir1, 'testfile1.py'), + os.path.join(tdir2, 'testfile2.py'), + ])) + + def test_find_with_excluded_hidden_dirs(self): + tdir1 = self._make_test_dir('.test1') + tdir2 = self._make_test_dir('test_2') + tdir3 = self._make_test_dir('test.3') + files = [ + os.path.join(tdir1, 'testfile1.py'), + os.path.join(tdir2, 'testfile2.py'), + os.path.join(tdir3, 'testfile3.py'), + ] + _touch_files(files) + actual = file_resources.GetCommandLineFiles([self.test_tmpdir], + recursive=True, + exclude=['*.test1*']) + + self.assertEqual( + sorted(actual), + sorted([ + os.path.join(tdir2, 'testfile2.py'), + os.path.join(tdir3, 'testfile3.py'), + ])) + + def test_find_with_excluded_hidden_dirs_relative(self): + """Test find with excluded hidden dirs. + + A regression test against a specific case where a hidden directory (one + beginning with a period) is being excluded, but it is also an immediate + child of the current directory which has been specified in a relative + manner. + + At its core, the bug has to do with overzealous stripping of "./foo" so that + it removes too much from "./.foo" . + """ + tdir1 = self._make_test_dir('.test1') + tdir2 = self._make_test_dir('test_2') + tdir3 = self._make_test_dir('test.3') + files = [ + os.path.join(tdir1, 'testfile1.py'), + os.path.join(tdir2, 'testfile2.py'), + os.path.join(tdir3, 'testfile3.py'), + ] + _touch_files(files) + + # We must temporarily change the current directory, so that we test against + # patterns like ./.test1/file instead of /tmp/foo/.test1/file + with _restore_working_dir(): + + os.chdir(self.test_tmpdir) + actual = file_resources.GetCommandLineFiles( + [os.path.relpath(self.test_tmpdir)], + recursive=True, + exclude=['*.test1*']) + + self.assertEqual( + sorted(actual), + sorted([ + os.path.join( + os.path.relpath(self.test_tmpdir), os.path.basename(tdir2), + 'testfile2.py'), + os.path.join( + os.path.relpath(self.test_tmpdir), os.path.basename(tdir3), + 'testfile3.py'), + ])) + + def test_find_with_excluded_dirs(self): + tdir1 = self._make_test_dir('test1') + tdir2 = self._make_test_dir('test2/testinner/') + tdir3 = self._make_test_dir('test3/foo/bar/bas/xxx') + files = [ + os.path.join(tdir1, 'testfile1.py'), + os.path.join(tdir2, 'testfile2.py'), + os.path.join(tdir3, 'testfile3.py'), + ] + _touch_files(files) -class IsPythonFileTest(unittest.TestCase): + os.chdir(self.test_tmpdir) - def setUp(self): + found = sorted( + file_resources.GetCommandLineFiles(['test1', 'test2', 'test3'], + recursive=True, + exclude=[ + 'test1', + 'test2/testinner/', + ])) + + self.assertEqual( + found, ['test3/foo/bar/bas/xxx/testfile3.py'.replace('/', os.path.sep)]) + + found = sorted( + file_resources.GetCommandLineFiles(['.'], + recursive=True, + exclude=[ + 'test1', + 'test3', + ])) + + self.assertEqual( + found, ['./test2/testinner/testfile2.py'.replace('/', os.path.sep)]) + + def test_find_with_excluded_current_dir(self): + with self.assertRaises(errors.YapfError): + file_resources.GetCommandLineFiles([], False, exclude=['./z']) + + +class IsPythonFileTest(yapf_test_helper.YAPFTest): + + def setUp(self): # pylint: disable=g-missing-super-call self.test_tmpdir = tempfile.mkdtemp() - def tearDown(self): + def tearDown(self): # pylint: disable=g-missing-super-call shutil.rmtree(self.test_tmpdir) def test_with_py_extension(self): @@ -162,32 +412,48 @@ def test_empty_without_py_extension(self): def test_python_shebang(self): file1 = os.path.join(self.test_tmpdir, 'testfile1') with open(file1, 'w') as f: - f.write(u'#!/usr/bin/python\n') + f.write('#!/usr/bin/python\n') self.assertTrue(file_resources.IsPythonFile(file1)) file2 = os.path.join(self.test_tmpdir, 'testfile2.run') with open(file2, 'w') as f: - f.write(u'#! /bin/python2\n') + f.write('#! /bin/python2\n') self.assertTrue(file_resources.IsPythonFile(file1)) def test_with_latin_encoding(self): file1 = os.path.join(self.test_tmpdir, 'testfile1') - with py3compat.open_with_encoding(file1, mode='w', encoding='latin-1') as f: - f.write(u'#! /bin/python2\n') + with codecs.open(file1, mode='w', encoding='latin-1') as f: + f.write('#! /bin/python2\n') self.assertTrue(file_resources.IsPythonFile(file1)) def test_with_invalid_encoding(self): file1 = os.path.join(self.test_tmpdir, 'testfile1') with open(file1, 'w') as f: - f.write(u'#! /bin/python2\n') - f.write(u'# -*- coding: iso-3-14159 -*-\n') + f.write('#! /bin/python2\n') + f.write('# -*- coding: iso-3-14159 -*-\n') self.assertFalse(file_resources.IsPythonFile(file1)) +class IsIgnoredTest(yapf_test_helper.YAPFTest): + + def test_root_path(self): + self.assertTrue(file_resources.IsIgnored('media', ['media'])) + self.assertFalse(file_resources.IsIgnored('media', ['media/*'])) + + def test_sub_path(self): + self.assertTrue(file_resources.IsIgnored('media/a', ['*/a'])) + self.assertTrue(file_resources.IsIgnored('media/b', ['media/*'])) + self.assertTrue(file_resources.IsIgnored('media/b/c', ['*/*/c'])) + + def test_trailing_slash(self): + self.assertTrue(file_resources.IsIgnored('z', ['z'])) + self.assertTrue(file_resources.IsIgnored('z', ['z' + os.path.sep])) + + class BufferedByteStream(object): def __init__(self): - self.stream = py3compat.BytesIO() + self.stream = BytesIO() def getvalue(self): # pylint: disable=invalid-name return self.stream.getvalue().decode('utf-8') @@ -197,42 +463,90 @@ def buffer(self): return self.stream -class WriteReformattedCodeTest(unittest.TestCase): +class WriteReformattedCodeTest(yapf_test_helper.YAPFTest): @classmethod - def setUpClass(cls): + def setUpClass(cls): # pylint: disable=g-missing-super-call cls.test_tmpdir = tempfile.mkdtemp() @classmethod - def tearDownClass(cls): + def tearDownClass(cls): # pylint: disable=g-missing-super-call shutil.rmtree(cls.test_tmpdir) def test_write_to_file(self): - s = u'foobar' - with tempfile.NamedTemporaryFile(dir=self.test_tmpdir) as testfile: + s = 'foobar\n' + with utils.NamedTempFile(dirname=self.test_tmpdir) as (f, fname): file_resources.WriteReformattedCode( - testfile.name, s, in_place=True, encoding='utf-8') - testfile.flush() + fname, s, in_place=True, encoding='utf-8') + f.flush() - with open(testfile.name) as f: - self.assertEqual(f.read(), s) + with open(fname) as f2: + self.assertEqual(f2.read(), s) def test_write_to_stdout(self): - s = u'foobar' - stream = BufferedByteStream() if py3compat.PY3 else py3compat.StringIO() - with stdout_redirector(stream): + s = 'foobar' + stream = BufferedByteStream() + with utils.stdout_redirector(stream): file_resources.WriteReformattedCode( None, s, in_place=False, encoding='utf-8') self.assertEqual(stream.getvalue(), s) def test_write_encoded_to_stdout(self): - s = '\ufeff# -*- coding: utf-8 -*-\nresult = "passed"\n' # pylint: disable=anomalous-unicode-escape-in-string - stream = BufferedByteStream() if py3compat.PY3 else py3compat.StringIO() - with stdout_redirector(stream): + s = '\ufeff# -*- coding: utf-8 -*-\nresult = "passed"\n' # pylint: disable=anomalous-unicode-escape-in-string # noqa + stream = BufferedByteStream() + with utils.stdout_redirector(stream): file_resources.WriteReformattedCode( None, s, in_place=False, encoding='utf-8') self.assertEqual(stream.getvalue(), s) +class LineEndingTest(yapf_test_helper.YAPFTest): + + def test_line_ending_linefeed(self): + lines = ['spam\n', 'spam\n'] + actual = file_resources.LineEnding(lines) + self.assertEqual(actual, '\n') + + def test_line_ending_carriage_return(self): + lines = ['spam\r', 'spam\r'] + actual = file_resources.LineEnding(lines) + self.assertEqual(actual, '\r') + + def test_line_ending_combo(self): + lines = ['spam\r\n', 'spam\r\n'] + actual = file_resources.LineEnding(lines) + self.assertEqual(actual, '\r\n') + + def test_line_ending_weighted(self): + lines = [ + 'spam\n', + 'spam\n', + 'spam\r', + 'spam\r\n', + ] + actual = file_resources.LineEnding(lines) + self.assertEqual(actual, '\n') + + def test_line_ending_empty(self): + lines = [] + actual = file_resources.LineEnding(lines) + self.assertEqual(actual, '\n') + + def test_line_ending_no_newline(self): + lines = ['spam'] + actual = file_resources.LineEnding(lines) + self.assertEqual(actual, '\n') + + def test_line_ending_tie(self): + lines = [ + 'spam\n', + 'spam\n', + 'spam\r\n', + 'spam\r\n', + ] + actual = file_resources.LineEnding(lines) + self.assertEqual(actual, '\n') + + if __name__ == '__main__': unittest.main() diff --git a/yapftests/format_decision_state_test.py b/yapftests/format_decision_state_test.py index bbdf2a168..4dd7b8b53 100644 --- a/yapftests/format_decision_state_test.py +++ b/yapftests/format_decision_state_test.py @@ -1,4 +1,4 @@ -# Copyright 2015-2016 Google Inc. All Rights Reserved. +# Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,151 +13,133 @@ # limitations under the License. """Tests for yapf.format_decision_state.""" -import sys import textwrap import unittest -from yapf.yapflib import comment_splicer +from yapf.pytree import pytree_utils from yapf.yapflib import format_decision_state -from yapf.yapflib import pytree_unwrapper -from yapf.yapflib import pytree_utils -from yapf.yapflib import pytree_visitor -from yapf.yapflib import split_penalty -from yapf.yapflib import subtype_assigner -from yapf.yapflib import unwrapped_line +from yapf.yapflib import logical_line +from yapf.yapflib import style +from yapftests import yapf_test_helper -class FormatDecisionStateTest(unittest.TestCase): - def _ParseAndUnwrap(self, code, dumptree=False): - """Produces unwrapped lines from the given code. +class FormatDecisionStateTest(yapf_test_helper.YAPFTest): - Parses the code into a tree, performs comment splicing and runs the - unwrapper. - - Arguments: - code: code to parse as a string - dumptree: if True, the parsed pytree (after comment splicing) is dumped - to stderr. Useful for debugging. - - Returns: - List of unwrapped lines. - """ - tree = pytree_utils.ParseCodeToTree(code) - comment_splicer.SpliceComments(tree) - subtype_assigner.AssignSubtypes(tree) - split_penalty.ComputeSplitPenalties(tree) - - if dumptree: - pytree_visitor.DumpPyTree(tree, target_stream=sys.stderr) - - return pytree_unwrapper.UnwrapPyTree(tree) - - def _FilterLine(self, uwline): - """Filter out nonsemantic tokens from the UnwrappedLines.""" - return [ft for ft in uwline.tokens - if ft.name not in pytree_utils.NONSEMANTIC_TOKENS] + @classmethod + def setUpClass(cls): + style.SetGlobalStyle(style.CreateYapfStyle()) def testSimpleFunctionDefWithNoSplitting(self): - code = textwrap.dedent(r""" - def f(a, b): - pass - """) - uwlines = self._ParseAndUnwrap(code) - uwline = unwrapped_line.UnwrappedLine(0, self._FilterLine(uwlines[0])) - uwline.CalculateFormattingInformation() + code = textwrap.dedent("""\ + def f(a, b): + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + lline = logical_line.LogicalLine(0, _FilterLine(llines[0])) + lline.CalculateFormattingInformation() # Add: 'f' - state = format_decision_state.FormatDecisionState(uwline, 0) + state = format_decision_state.FormatDecisionState(lline, 0) + state.MoveStateToNextToken() self.assertEqual('f', state.next_token.value) - self.assertFalse(state.CanSplit()) + self.assertFalse(state.CanSplit(False)) # Add: '(' state.AddTokenToState(False, True) self.assertEqual('(', state.next_token.value) - self.assertFalse(state.CanSplit()) + self.assertFalse(state.CanSplit(False)) self.assertFalse(state.MustSplit()) # Add: 'a' state.AddTokenToState(False, True) self.assertEqual('a', state.next_token.value) - self.assertTrue(state.CanSplit()) + self.assertTrue(state.CanSplit(False)) self.assertFalse(state.MustSplit()) # Add: ',' state.AddTokenToState(False, True) self.assertEqual(',', state.next_token.value) - self.assertFalse(state.CanSplit()) + self.assertFalse(state.CanSplit(False)) self.assertFalse(state.MustSplit()) # Add: 'b' state.AddTokenToState(False, True) self.assertEqual('b', state.next_token.value) - self.assertTrue(state.CanSplit()) + self.assertTrue(state.CanSplit(False)) self.assertFalse(state.MustSplit()) # Add: ')' state.AddTokenToState(False, True) self.assertEqual(')', state.next_token.value) - self.assertTrue(state.CanSplit()) + self.assertTrue(state.CanSplit(False)) self.assertFalse(state.MustSplit()) # Add: ':' state.AddTokenToState(False, True) self.assertEqual(':', state.next_token.value) - self.assertFalse(state.CanSplit()) + self.assertFalse(state.CanSplit(False)) self.assertFalse(state.MustSplit()) clone = state.Clone() self.assertEqual(repr(state), repr(clone)) def testSimpleFunctionDefWithSplitting(self): - code = textwrap.dedent(r""" - def f(a, b): - pass - """) - uwlines = self._ParseAndUnwrap(code) - uwline = unwrapped_line.UnwrappedLine(0, self._FilterLine(uwlines[0])) - uwline.CalculateFormattingInformation() + code = textwrap.dedent("""\ + def f(a, b): + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + lline = logical_line.LogicalLine(0, _FilterLine(llines[0])) + lline.CalculateFormattingInformation() # Add: 'f' - state = format_decision_state.FormatDecisionState(uwline, 0) + state = format_decision_state.FormatDecisionState(lline, 0) + state.MoveStateToNextToken() self.assertEqual('f', state.next_token.value) - self.assertFalse(state.CanSplit()) + self.assertFalse(state.CanSplit(False)) # Add: '(' state.AddTokenToState(True, True) self.assertEqual('(', state.next_token.value) - self.assertFalse(state.CanSplit()) + self.assertFalse(state.CanSplit(False)) # Add: 'a' state.AddTokenToState(True, True) self.assertEqual('a', state.next_token.value) - self.assertTrue(state.CanSplit()) + self.assertTrue(state.CanSplit(False)) # Add: ',' state.AddTokenToState(True, True) self.assertEqual(',', state.next_token.value) - self.assertFalse(state.CanSplit()) + self.assertFalse(state.CanSplit(False)) # Add: 'b' state.AddTokenToState(True, True) self.assertEqual('b', state.next_token.value) - self.assertTrue(state.CanSplit()) + self.assertTrue(state.CanSplit(False)) # Add: ')' state.AddTokenToState(True, True) self.assertEqual(')', state.next_token.value) - self.assertTrue(state.CanSplit()) + self.assertTrue(state.CanSplit(False)) # Add: ':' state.AddTokenToState(True, True) self.assertEqual(':', state.next_token.value) - self.assertFalse(state.CanSplit()) + self.assertFalse(state.CanSplit(False)) clone = state.Clone() self.assertEqual(repr(state), repr(clone)) +def _FilterLine(lline): + """Filter out nonsemantic tokens from the LogicalLines.""" + return [ + ft for ft in lline.tokens + if ft.name not in pytree_utils.NONSEMANTIC_TOKENS + ] + + if __name__ == '__main__': unittest.main() diff --git a/yapftests/format_token_test.py b/yapftests/format_token_test.py index b8078c733..0db5680ab 100644 --- a/yapftests/format_token_test.py +++ b/yapftests/format_token_test.py @@ -1,4 +1,4 @@ -# Copyright 2015-2016 Google Inc. All Rights Reserved. +# Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,22 +15,82 @@ import unittest -from lib2to3 import pytree -from lib2to3.pgen2 import token +from yapf_third_party._ylib2to3 import pytree +from yapf_third_party._ylib2to3.pgen2 import token + from yapf.yapflib import format_token +from yapftests import yapf_test_helper + + +class TabbedContinuationAlignPaddingTest(yapf_test_helper.YAPFTest): + + def testSpace(self): + align_style = 'SPACE' + + pad = format_token._TabbedContinuationAlignPadding(0, align_style, 2) + self.assertEqual(pad, '') + + pad = format_token._TabbedContinuationAlignPadding(2, align_style, 2) + self.assertEqual(pad, ' ' * 2) + + pad = format_token._TabbedContinuationAlignPadding(5, align_style, 2) + self.assertEqual(pad, ' ' * 5) + + def testFixed(self): + align_style = 'FIXED' + + pad = format_token._TabbedContinuationAlignPadding(0, align_style, 4) + self.assertEqual(pad, '') + + pad = format_token._TabbedContinuationAlignPadding(2, align_style, 4) + self.assertEqual(pad, '\t') -class FormatTokenTest(unittest.TestCase): + pad = format_token._TabbedContinuationAlignPadding(5, align_style, 4) + self.assertEqual(pad, '\t' * 2) + + def testVAlignRight(self): + align_style = 'VALIGN-RIGHT' + + pad = format_token._TabbedContinuationAlignPadding(0, align_style, 4) + self.assertEqual(pad, '') + + pad = format_token._TabbedContinuationAlignPadding(2, align_style, 4) + self.assertEqual(pad, '\t') + + pad = format_token._TabbedContinuationAlignPadding(4, align_style, 4) + self.assertEqual(pad, '\t') + + pad = format_token._TabbedContinuationAlignPadding(5, align_style, 4) + self.assertEqual(pad, '\t' * 2) + + +class FormatTokenTest(yapf_test_helper.YAPFTest): def testSimple(self): - tok = format_token.FormatToken(pytree.Leaf(token.STRING, "'hello world'")) - self.assertEqual("FormatToken(name=STRING, value='hello world')", str(tok)) + tok = format_token.FormatToken( + pytree.Leaf(token.STRING, "'hello world'"), 'STRING') + self.assertEqual( + "FormatToken(name=DOCSTRING, value='hello world', column=0, " + 'lineno=0, splitpenalty=0)', str(tok)) self.assertTrue(tok.is_string) - tok = format_token.FormatToken(pytree.Leaf(token.COMMENT, "# A comment")) - self.assertEqual("FormatToken(name=COMMENT, value=# A comment)", str(tok)) + tok = format_token.FormatToken( + pytree.Leaf(token.COMMENT, '# A comment'), 'COMMENT') + self.assertEqual( + 'FormatToken(name=COMMENT, value=# A comment, column=0, ' + 'lineno=0, splitpenalty=0)', str(tok)) self.assertTrue(tok.is_comment) + def testIsMultilineString(self): + tok = format_token.FormatToken( + pytree.Leaf(token.STRING, '"""hello"""'), 'STRING') + self.assertTrue(tok.is_multiline_string) + + tok = format_token.FormatToken( + pytree.Leaf(token.STRING, 'r"""hello"""'), 'STRING') + self.assertTrue(tok.is_multiline_string) + -if __name__ == "__main__": +if __name__ == '__main__': unittest.main() diff --git a/yapftests/line_joiner_test.py b/yapftests/line_joiner_test.py index 842f87f10..01dd479b2 100644 --- a/yapftests/line_joiner_test.py +++ b/yapftests/line_joiner_test.py @@ -1,4 +1,4 @@ -# Copyright 2015-2016 Google Inc. All Rights Reserved. +# Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,85 +16,65 @@ import textwrap import unittest -from yapf.yapflib import comment_splicer from yapf.yapflib import line_joiner -from yapf.yapflib import pytree_unwrapper -from yapf.yapflib import pytree_utils from yapf.yapflib import style +from yapftests import yapf_test_helper -class LineJoinerTest(unittest.TestCase): + +class LineJoinerTest(yapf_test_helper.YAPFTest): @classmethod def setUpClass(cls): style.SetGlobalStyle(style.CreatePEP8Style()) - def _ParseAndUnwrap(self, code): - """Produces unwrapped lines from the given code. - - Parses the code into a tree, performs comment splicing and runs the - unwrapper. - - Arguments: - code: code to parse as a string - - Returns: - List of unwrapped lines. - """ - tree = pytree_utils.ParseCodeToTree(code) - comment_splicer.SpliceComments(tree) - uwlines = pytree_unwrapper.UnwrapPyTree(tree) - for uwl in uwlines: - uwl.CalculateFormattingInformation() - return uwlines - def _CheckLineJoining(self, code, join_lines): - """Check that the given UnwrappedLines are joined as expected. + """Check that the given LogicalLines are joined as expected. Arguments: code: The code to check to see if we can join it. join_lines: True if we expect the lines to be joined. """ - uwlines = self._ParseAndUnwrap(code) - self.assertEqual(line_joiner.CanMergeMultipleLines(uwlines), join_lines) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(line_joiner.CanMergeMultipleLines(llines), join_lines) def testSimpleSingleLineStatement(self): - code = textwrap.dedent(u"""\ + code = textwrap.dedent("""\ if isinstance(a, int): continue - """) + """) self._CheckLineJoining(code, join_lines=True) def testSimpleMultipleLineStatement(self): - code = textwrap.dedent(u"""\ + code = textwrap.dedent("""\ if isinstance(b, int): continue - """) + """) self._CheckLineJoining(code, join_lines=False) def testSimpleMultipleLineComplexStatement(self): - code = textwrap.dedent(u"""\ + code = textwrap.dedent("""\ if isinstance(c, int): while True: continue - """) + """) self._CheckLineJoining(code, join_lines=False) def testSimpleMultipleLineStatementWithComment(self): - code = textwrap.dedent(u"""\ + code = textwrap.dedent("""\ if isinstance(d, int): continue # We're pleased that d's an int. - """) + """) self._CheckLineJoining(code, join_lines=True) def testSimpleMultipleLineStatementWithLargeIndent(self): - code = textwrap.dedent(u"""\ + code = textwrap.dedent("""\ if isinstance(e, int): continue - """) + """) self._CheckLineJoining(code, join_lines=True) def testOverColumnLimit(self): - code = textwrap.dedent(u"""\ + code = textwrap.dedent("""\ if instance(bbbbbbbbbbbbbbbbbbbbbbbbb, int): cccccccccccccccccccccccccc = ddddddddddddddddddddd - """) + """) # noqa self._CheckLineJoining(code, join_lines=False) diff --git a/yapftests/logical_line_test.py b/yapftests/logical_line_test.py new file mode 100644 index 000000000..2526b591c --- /dev/null +++ b/yapftests/logical_line_test.py @@ -0,0 +1,93 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for yapf.logical_line.""" + +import textwrap +import unittest + +from yapf_third_party._ylib2to3 import pytree +from yapf_third_party._ylib2to3.pgen2 import token + +from yapf.pytree import split_penalty +from yapf.yapflib import format_token +from yapf.yapflib import logical_line + +from yapftests import yapf_test_helper + + +class LogicalLineBasicTest(yapf_test_helper.YAPFTest): + + def testConstruction(self): + toks = _MakeFormatTokenList([(token.DOT, '.', 'DOT'), + (token.VBAR, '|', 'VBAR')]) + lline = logical_line.LogicalLine(20, toks) + self.assertEqual(20, lline.depth) + self.assertEqual(['DOT', 'VBAR'], [tok.name for tok in lline.tokens]) + + def testFirstLast(self): + toks = _MakeFormatTokenList([(token.DOT, '.', 'DOT'), + (token.LPAR, '(', 'LPAR'), + (token.VBAR, '|', 'VBAR')]) + lline = logical_line.LogicalLine(20, toks) + self.assertEqual(20, lline.depth) + self.assertEqual('DOT', lline.first.name) + self.assertEqual('VBAR', lline.last.name) + + def testAsCode(self): + toks = _MakeFormatTokenList([(token.DOT, '.', 'DOT'), + (token.LPAR, '(', 'LPAR'), + (token.VBAR, '|', 'VBAR')]) + lline = logical_line.LogicalLine(2, toks) + self.assertEqual(' . ( |', lline.AsCode()) + + def testAppendToken(self): + lline = logical_line.LogicalLine(0) + lline.AppendToken(_MakeFormatTokenLeaf(token.LPAR, '(', 'LPAR')) + lline.AppendToken(_MakeFormatTokenLeaf(token.RPAR, ')', 'RPAR')) + self.assertEqual(['LPAR', 'RPAR'], [tok.name for tok in lline.tokens]) + + +class LogicalLineFormattingInformationTest(yapf_test_helper.YAPFTest): + + def testFuncDef(self): + code = textwrap.dedent("""\ + def f(a, b): + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + + f = llines[0].tokens[1] + self.assertFalse(f.can_break_before) + self.assertFalse(f.must_break_before) + self.assertEqual(f.split_penalty, split_penalty.UNBREAKABLE) + + lparen = llines[0].tokens[2] + self.assertFalse(lparen.can_break_before) + self.assertFalse(lparen.must_break_before) + self.assertEqual(lparen.split_penalty, split_penalty.UNBREAKABLE) + + +def _MakeFormatTokenLeaf(token_type, token_value, name): + return format_token.FormatToken(pytree.Leaf(token_type, token_value), name) + + +def _MakeFormatTokenList(token_type_values): + return [ + _MakeFormatTokenLeaf(token_type, token_value, token_name) + for token_type, token_value, token_name in token_type_values + ] + + +if __name__ == '__main__': + unittest.main() diff --git a/yapftests/main_test.py b/yapftests/main_test.py index e5a5a74ce..b5d9b926e 100644 --- a/yapftests/main_test.py +++ b/yapftests/main_test.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2015-2016 Google Inc. All Rights Reserved. +# Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,21 +14,49 @@ # limitations under the License. """Tests for yapf.__init__.main.""" -from contextlib import contextmanager import sys import unittest +from contextlib import contextmanager +from io import StringIO + import yapf -try: - from StringIO import StringIO -except ImportError: # Python 3 - # Note: io.StringIO is different in Python 2, so try for python 2 first. - from io import StringIO +from yapftests import yapf_test_helper + + +class IO(object): + """IO is a thin wrapper around StringIO. + + This is strictly to wrap the Python 3 StringIO object so that it can supply a + "buffer" attribute. + """ + + class Buffer(object): + + def __init__(self): + self.string_io = StringIO() + + def write(self, s): + if isinstance(s, bytes): + s = str(s, 'utf-8') + self.string_io.write(s) + + def getvalue(self): + return self.string_io.getvalue() + + def __init__(self): + self.buffer = self.Buffer() + + def write(self, s): + self.buffer.write(s) + + def getvalue(self): + return self.buffer.getvalue() @contextmanager def captured_output(): - new_out, new_err = StringIO(), StringIO() + new_out, new_err = IO(), IO() old_out, old_err = sys.stdout, sys.stderr try: sys.stdout, sys.stderr = new_out, new_err @@ -39,7 +67,7 @@ def captured_output(): @contextmanager def patched_input(code): - "Monkey patch code as though it were coming from stdin." + """Monkey patch code as though it were coming from stdin.""" def lines(): for line in code.splitlines(): @@ -50,18 +78,18 @@ def patch_raw_input(lines=lines()): return next(lines) try: - raw_input = yapf.py3compat.raw_input - yapf.py3compat.raw_input = patch_raw_input + orig_raw_import = yapf._raw_input + yapf._raw_input = patch_raw_input yield finally: - yapf.py3compat.raw_input = raw_input + yapf._raw_input = orig_raw_import -class RunMainTest(unittest.TestCase): +class RunMainTest(yapf_test_helper.YAPFTest): def testShouldHandleYapfError(self): - """run_main should handle YapfError and sys.exit(1)""" - expected_message = 'yapf: Input filenames did not match any python files\n' + """run_main should handle YapfError and sys.exit(1).""" + expected_message = 'yapf: input filenames did not match any python files\n' sys.argv = ['yapf', 'foo.c'] with captured_output() as (out, err): with self.assertRaises(SystemExit): @@ -70,49 +98,42 @@ def testShouldHandleYapfError(self): self.assertEqual(err.getvalue(), expected_message) -class MainTest(unittest.TestCase): +class MainTest(yapf_test_helper.YAPFTest): def testNoPythonFilesMatched(self): - with self.assertRaisesRegexp(yapf.errors.YapfError, - 'did not match any python files'): + with self.assertRaisesRegex(yapf.errors.YapfError, + 'did not match any python files'): yapf.main(['yapf', 'foo.c']) def testEchoInput(self): - code = "a = 1\nb = 2\n" + code = 'a = 1\nb = 2\n' with patched_input(code): - with captured_output() as (out, err): + with captured_output() as (out, _): ret = yapf.main([]) self.assertEqual(ret, 0) self.assertEqual(out.getvalue(), code) def testEchoInputWithStyle(self): - code = "def f(a = 1):\n return 2*a\n" - chromium_code = "def f(a=1):\n return 2 * a\n" + code = 'def f(a = 1\n\n):\n return 2*a\n' + yapf_code = 'def f(a=1):\n return 2 * a\n' with patched_input(code): - with captured_output() as (out, err): - ret = yapf.main(['-', '--style=chromium']) - self.assertEqual(ret, 2) - self.assertEqual(out.getvalue(), chromium_code) + with captured_output() as (out, _): + ret = yapf.main(['-', '--style=yapf']) + self.assertEqual(ret, 0) + self.assertEqual(out.getvalue(), yapf_code) def testEchoBadInput(self): - bad_syntax = " a = 1\n" + bad_syntax = ' a = 1\n' with patched_input(bad_syntax): - with captured_output() as (out, err): - with self.assertRaisesRegexp(SyntaxError, "unexpected indent"): + with captured_output() as (_, _): + with self.assertRaisesRegex(yapf.errors.YapfError, 'unexpected indent'): yapf.main([]) def testHelp(self): - with captured_output() as (out, err): + with captured_output() as (out, _): ret = yapf.main(['-', '--style-help', '--style=pep8']) self.assertEqual(ret, 0) help_message = out.getvalue() - self.assertIn("INDENT_WIDTH=4", help_message) - self.assertIn("The number of spaces required before a trailing comment.", + self.assertIn('indent_width=4', help_message) + self.assertIn('The number of spaces required before a trailing comment.', help_message) - - def testVersion(self): - with captured_output() as (out, err): - ret = yapf.main(['-', '--version']) - self.assertEqual(ret, 0) - version = 'yapf {}\n'.format(yapf.__version__) - self.assertEqual(version, out.getvalue()) diff --git a/yapftests/pytree_unwrapper_test.py b/yapftests/pytree_unwrapper_test.py index 057e03753..b297560d8 100644 --- a/yapftests/pytree_unwrapper_test.py +++ b/yapftests/pytree_unwrapper_test.py @@ -1,4 +1,4 @@ -# Copyright 2015-2016 Google Inc. All Rights Reserved. +# Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,194 +13,182 @@ # limitations under the License. """Tests for yapf.pytree_unwrapper.""" -import sys import textwrap import unittest -from yapf.yapflib import comment_splicer -from yapf.yapflib import pytree_unwrapper -from yapf.yapflib import pytree_utils -from yapf.yapflib import pytree_visitor +from yapf.pytree import pytree_utils +from yapftests import yapf_test_helper -class PytreeUnwrapperTest(unittest.TestCase): - def _ParseAndUnwrap(self, code, dumptree=False): - """Produces unwrapped lines from the given code. +class PytreeUnwrapperTest(yapf_test_helper.YAPFTest): - Parses the code into a tree, performs comment splicing and runs the - unwrapper. - - Arguments: - code: code to parse as a string - dumptree: if True, the parsed pytree (after comment splicing) is dumped - to stderr. Useful for debugging. - - Returns: - List of unwrapped lines. - """ - tree = pytree_utils.ParseCodeToTree(code) - comment_splicer.SpliceComments(tree) - - if dumptree: - pytree_visitor.DumpPyTree(tree, target_stream=sys.stderr) - - return pytree_unwrapper.UnwrapPyTree(tree) - - def _CheckUnwrappedLines(self, uwlines, list_of_expected): - """Check that the given UnwrappedLines match expectations. + def _CheckLogicalLines(self, llines, list_of_expected): + """Check that the given LogicalLines match expectations. Args: - uwlines: list of UnwrappedLine + llines: list of LogicalLine list_of_expected: list of (depth, values) pairs. Non-semantic tokens are filtered out from the expected values. """ actual = [] - for uwl in uwlines: - filtered_values = [ft.value for ft in uwl.tokens - if ft.name not in pytree_utils.NONSEMANTIC_TOKENS] - actual.append((uwl.depth, filtered_values)) + for lline in llines: + filtered_values = [ + ft.value + for ft in lline.tokens + if ft.name not in pytree_utils.NONSEMANTIC_TOKENS + ] + actual.append((lline.depth, filtered_values)) self.assertEqual(list_of_expected, actual) def testSimpleFileScope(self): - code = textwrap.dedent(r""" - x = 1 - # a comment - y = 2 - """) - uwlines = self._ParseAndUnwrap(code) - self._CheckUnwrappedLines(uwlines, [ + code = textwrap.dedent("""\ + x = 1 + # a comment + y = 2 + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckLogicalLines(llines, [ (0, ['x', '=', '1']), (0, ['# a comment']), - (0, ['y', '=', '2'])]) # yapf: disable + (0, ['y', '=', '2']), + ]) def testSimpleMultilineStatement(self): - code = textwrap.dedent(r""" - y = (1 + - x) - """) - uwlines = self._ParseAndUnwrap(code) - self._CheckUnwrappedLines(uwlines, [ - (0, ['y', '=', '(', '1', '+', 'x', ')'])]) # yapf: disable + code = textwrap.dedent("""\ + y = (1 + + x) + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckLogicalLines(llines, [ + (0, ['y', '=', '(', '1', '+', 'x', ')']), + ]) def testFileScopeWithInlineComment(self): - code = textwrap.dedent(r""" - x = 1 # a comment - y = 2 - """) - uwlines = self._ParseAndUnwrap(code) - self._CheckUnwrappedLines(uwlines, [ + code = textwrap.dedent("""\ + x = 1 # a comment + y = 2 + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckLogicalLines(llines, [ (0, ['x', '=', '1', '# a comment']), - (0, ['y', '=', '2'])]) # yapf: disable + (0, ['y', '=', '2']), + ]) def testSimpleIf(self): - code = textwrap.dedent(r""" - if foo: - x = 1 - y = 2 - """) - uwlines = self._ParseAndUnwrap(code) - self._CheckUnwrappedLines(uwlines, [ + code = textwrap.dedent("""\ + if foo: + x = 1 + y = 2 + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckLogicalLines(llines, [ (0, ['if', 'foo', ':']), (1, ['x', '=', '1']), - (1, ['y', '=', '2'])]) # yapf: disable + (1, ['y', '=', '2']), + ]) def testSimpleIfWithComments(self): - code = textwrap.dedent(r""" - # c1 - if foo: # c2 - x = 1 - y = 2 - """) - uwlines = self._ParseAndUnwrap(code) - self._CheckUnwrappedLines(uwlines, [ + code = textwrap.dedent("""\ + # c1 + if foo: # c2 + x = 1 + y = 2 + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckLogicalLines(llines, [ (0, ['# c1']), (0, ['if', 'foo', ':', '# c2']), (1, ['x', '=', '1']), - (1, ['y', '=', '2'])]) # yapf: disable + (1, ['y', '=', '2']), + ]) def testIfWithCommentsInside(self): - code = textwrap.dedent(r""" - if foo: - # c1 - x = 1 # c2 - # c3 - y = 2 - """) - uwlines = self._ParseAndUnwrap(code) - self._CheckUnwrappedLines(uwlines, [ + code = textwrap.dedent("""\ + if foo: + # c1 + x = 1 # c2 + # c3 + y = 2 + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckLogicalLines(llines, [ (0, ['if', 'foo', ':']), (1, ['# c1']), (1, ['x', '=', '1', '# c2']), (1, ['# c3']), - (1, ['y', '=', '2'])]) # yapf: disable + (1, ['y', '=', '2']), + ]) def testIfElifElse(self): - code = textwrap.dedent(r""" - if x: - x = 1 # c1 - elif y: # c2 - y = 1 - else: - # c3 - z = 1 - """) - uwlines = self._ParseAndUnwrap(code) - self._CheckUnwrappedLines(uwlines, [ + code = textwrap.dedent("""\ + if x: + x = 1 # c1 + elif y: # c2 + y = 1 + else: + # c3 + z = 1 + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckLogicalLines(llines, [ (0, ['if', 'x', ':']), (1, ['x', '=', '1', '# c1']), (0, ['elif', 'y', ':', '# c2']), (1, ['y', '=', '1']), (0, ['else', ':']), (1, ['# c3']), - (1, ['z', '=', '1'])]) # yapf: disable + (1, ['z', '=', '1']), + ]) def testNestedCompoundTwoLevel(self): - code = textwrap.dedent(r""" - if x: - x = 1 # c1 - while t: - # c2 - j = 1 - k = 1 - """) - uwlines = self._ParseAndUnwrap(code) - self._CheckUnwrappedLines(uwlines, [ + code = textwrap.dedent("""\ + if x: + x = 1 # c1 + while t: + # c2 + j = 1 + k = 1 + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckLogicalLines(llines, [ (0, ['if', 'x', ':']), (1, ['x', '=', '1', '# c1']), (1, ['while', 't', ':']), (2, ['# c2']), (2, ['j', '=', '1']), - (1, ['k', '=', '1'])]) # yapf: disable + (1, ['k', '=', '1']), + ]) def testSimpleWhile(self): - code = textwrap.dedent(r""" - while x > 1: # c1 - # c2 - x = 1 - """) - uwlines = self._ParseAndUnwrap(code) - self._CheckUnwrappedLines(uwlines, [ + code = textwrap.dedent("""\ + while x > 1: # c1 + # c2 + x = 1 + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckLogicalLines(llines, [ (0, ['while', 'x', '>', '1', ':', '# c1']), (1, ['# c2']), - (1, ['x', '=', '1'])]) # yapf: disable + (1, ['x', '=', '1']), + ]) def testSimpleTry(self): - code = textwrap.dedent(r""" - try: - pass - except: - pass - except: - pass - else: - pass - finally: - pass - """) - uwlines = self._ParseAndUnwrap(code) - self._CheckUnwrappedLines(uwlines, [ + code = textwrap.dedent("""\ + try: + pass + except: + pass + except: + pass + else: + pass + finally: + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckLogicalLines(llines, [ (0, ['try', ':']), (1, ['pass']), (0, ['except', ':']), @@ -210,172 +198,158 @@ def testSimpleTry(self): (0, ['else', ':']), (1, ['pass']), (0, ['finally', ':']), - (1, ['pass'])]) # yapf: disable + (1, ['pass']), + ]) def testSimpleFuncdef(self): - code = textwrap.dedent(r""" - def foo(x): # c1 - # c2 - return x - """) - uwlines = self._ParseAndUnwrap(code) - self._CheckUnwrappedLines(uwlines, [ + code = textwrap.dedent("""\ + def foo(x): # c1 + # c2 + return x + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckLogicalLines(llines, [ (0, ['def', 'foo', '(', 'x', ')', ':', '# c1']), (1, ['# c2']), - (1, ['return', 'x'])]) # yapf: disable + (1, ['return', 'x']), + ]) def testTwoFuncDefs(self): - code = textwrap.dedent(r""" - def foo(x): # c1 - # c2 - return x - - def bar(): # c3 - # c4 - return x - """) - uwlines = self._ParseAndUnwrap(code) - self._CheckUnwrappedLines(uwlines, [ + code = textwrap.dedent("""\ + def foo(x): # c1 + # c2 + return x + + def bar(): # c3 + # c4 + return x + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckLogicalLines(llines, [ (0, ['def', 'foo', '(', 'x', ')', ':', '# c1']), (1, ['# c2']), (1, ['return', 'x']), (0, ['def', 'bar', '(', ')', ':', '# c3']), (1, ['# c4']), - (1, ['return', 'x'])]) # yapf: disable + (1, ['return', 'x']), + ]) def testSimpleClassDef(self): - code = textwrap.dedent(r""" - class Klass: # c1 - # c2 - p = 1 - """) - uwlines = self._ParseAndUnwrap(code) - self._CheckUnwrappedLines(uwlines, [ + code = textwrap.dedent("""\ + class Klass: # c1 + # c2 + p = 1 + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckLogicalLines(llines, [ (0, ['class', 'Klass', ':', '# c1']), (1, ['# c2']), - (1, ['p', '=', '1'])]) # yapf: disable + (1, ['p', '=', '1']), + ]) def testSingleLineStmtInFunc(self): - code = textwrap.dedent(r""" + code = textwrap.dedent("""\ def f(): return 37 - """) - uwlines = self._ParseAndUnwrap(code) - self._CheckUnwrappedLines(uwlines, [ + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckLogicalLines(llines, [ (0, ['def', 'f', '(', ')', ':']), - (1, ['return', '37'])]) # yapf: disable + (1, ['return', '37']), + ]) def testMultipleComments(self): - code = textwrap.dedent(r""" + code = textwrap.dedent("""\ # Comment #1 # Comment #2 def f(): pass - """) - uwlines = self._ParseAndUnwrap(code) - self._CheckUnwrappedLines(uwlines, [ + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckLogicalLines(llines, [ (0, ['# Comment #1']), (0, ['# Comment #2']), (0, ['def', 'f', '(', ')', ':']), - (1, ['pass'])]) # yapf: disable + (1, ['pass']), + ]) def testSplitListWithComment(self): - code = textwrap.dedent(r""" - a = [ - 'a', - 'b', - 'c', # hello world - ] - """) - uwlines = self._ParseAndUnwrap(code) - self._CheckUnwrappedLines(uwlines, [ - (0, ['a', '=', '[', "'a'", ',', "'b'", ',', - "'c'", ',', '# hello world', ']'])]) # yapf: disable - - -class MatchBracketsTest(unittest.TestCase): - - def _ParseAndUnwrap(self, code, dumptree=False): - """Produces unwrapped lines from the given code. - - Parses the code into a tree, match brackets and runs the unwrapper. - - Arguments: - code: code to parse as a string - dumptree: if True, the parsed pytree (after comment splicing) is dumped to - stderr. Useful for debugging. - - Returns: - List of unwrapped lines. - """ - tree = pytree_utils.ParseCodeToTree(code) - comment_splicer.SpliceComments(tree) + code = textwrap.dedent("""\ + a = [ + 'a', + 'b', + 'c', # hello world + ] + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckLogicalLines(llines, [(0, [ + 'a', '=', '[', "'a'", ',', "'b'", ',', "'c'", ',', '# hello world', ']' + ])]) - if dumptree: - pytree_visitor.DumpPyTree(tree, target_stream=sys.stderr) - return pytree_unwrapper.UnwrapPyTree(tree) +class MatchBracketsTest(yapf_test_helper.YAPFTest): - def _CheckMatchingBrackets(self, uwlines, list_of_expected): + def _CheckMatchingBrackets(self, llines, list_of_expected): """Check that the tokens have the expected matching bracket. Arguments: - uwlines: list of UnwrappedLine. + llines: list of LogicalLine. list_of_expected: list of (index, index) pairs. The matching brackets at the indexes need to match. Non-semantic tokens are filtered out from the expected values. """ actual = [] - for uwl in uwlines: - filtered_values = [(ft, ft.matching_bracket) for ft in uwl.tokens + for lline in llines: + filtered_values = [(ft, ft.matching_bracket) + for ft in lline.tokens if ft.name not in pytree_utils.NONSEMANTIC_TOKENS] if filtered_values: actual.append(filtered_values) for index, bracket_list in enumerate(list_of_expected): - uwline = actual[index] + lline = actual[index] if not bracket_list: - for value in uwline: + for value in lline: self.assertIsNone(value[1]) else: for open_bracket, close_bracket in bracket_list: - self.assertEqual(uwline[open_bracket][0], uwline[close_bracket][1]) - self.assertEqual(uwline[close_bracket][0], uwline[open_bracket][1]) + self.assertEqual(lline[open_bracket][0], lline[close_bracket][1]) + self.assertEqual(lline[close_bracket][0], lline[open_bracket][1]) def testFunctionDef(self): code = textwrap.dedent("""\ - def foo(a, b={'hello': ['w','d']}, c=[42, 37]): + def foo(a, b=['w','d'], c=[42, 37]): pass - """) - uwlines = self._ParseAndUnwrap(code) - self._CheckMatchingBrackets(uwlines, [ - [(2, 24), (7, 15), (10, 14), (19, 23)], - [] - ]) # yapf: disable + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckMatchingBrackets(llines, [ + [(2, 20), (7, 11), (15, 19)], + [], + ]) def testDecorator(self): code = textwrap.dedent("""\ @bar() def foo(a, b, c): pass - """) - uwlines = self._ParseAndUnwrap(code) - self._CheckMatchingBrackets(uwlines, [ + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckMatchingBrackets(llines, [ [(2, 3)], [(2, 8)], - [] - ]) # yapf: disable + [], + ]) def testClassDef(self): code = textwrap.dedent("""\ class A(B, C, D): pass - """) - uwlines = self._ParseAndUnwrap(code) - self._CheckMatchingBrackets(uwlines, [ + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckMatchingBrackets(llines, [ [(2, 8)], - [] - ]) # yapf: disable + [], + ]) if __name__ == '__main__': diff --git a/yapftests/pytree_utils_test.py b/yapftests/pytree_utils_test.py index 6c650c046..fe31eb83c 100644 --- a/yapftests/pytree_utils_test.py +++ b/yapftests/pytree_utils_test.py @@ -1,4 +1,4 @@ -# Copyright 2015-2016 Google Inc. All Rights Reserved. +# Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,11 +15,13 @@ import unittest -from lib2to3 import pygram -from lib2to3 import pytree -from lib2to3.pgen2 import token +from yapf_third_party._ylib2to3 import pygram +from yapf_third_party._ylib2to3 import pytree +from yapf_third_party._ylib2to3.pgen2 import token -from yapf.yapflib import pytree_utils +from yapf.pytree import pytree_utils + +from yapftests import yapf_test_helper # More direct access to the symbol->number mapping living within the grammar # module. @@ -33,7 +35,7 @@ _FOO5 = 'foo5' -class NodeNameTest(unittest.TestCase): +class NodeNameTest(yapf_test_helper.YAPFTest): def testNodeNameForLeaf(self): leaf = pytree.Leaf(token.LPAR, '(') @@ -45,7 +47,7 @@ def testNodeNameForNode(self): self.assertEqual('suite', pytree_utils.NodeName(node)) -class ParseCodeToTreeTest(unittest.TestCase): +class ParseCodeToTreeTest(yapf_test_helper.YAPFTest): def testParseCodeToTree(self): # Since ParseCodeToTree is a thin wrapper around underlying lib2to3 @@ -63,19 +65,15 @@ def testPrintFunctionToTree(self): self.assertEqual('simple_stmt', pytree_utils.NodeName(tree.children[0])) def testPrintStatementToTree(self): - tree = pytree_utils.ParseCodeToTree('print "hello world"\n') - self.assertEqual('file_input', pytree_utils.NodeName(tree)) - self.assertEqual(2, len(tree.children)) - self.assertEqual('simple_stmt', pytree_utils.NodeName(tree.children[0])) + with self.assertRaises(SyntaxError): + pytree_utils.ParseCodeToTree('print "hello world"\n') def testClassNotLocal(self): - tree = pytree_utils.ParseCodeToTree('class nonlocal: pass\n') - self.assertEqual('file_input', pytree_utils.NodeName(tree)) - self.assertEqual(2, len(tree.children)) - self.assertEqual('classdef', pytree_utils.NodeName(tree.children[0])) + with self.assertRaises(SyntaxError): + pytree_utils.ParseCodeToTree('class nonlocal: pass\n') -class InsertNodesBeforeAfterTest(unittest.TestCase): +class InsertNodesBeforeAfterTest(yapf_test_helper.YAPFTest): def _BuildSimpleTree(self): # Builds a simple tree we can play with in the tests. @@ -91,8 +89,8 @@ def _BuildSimpleTree(self): lpar2 = pytree.Leaf(token.LPAR, '(') simple_stmt = pytree.Node(_GRAMMAR_SYMBOL2NUMBER['simple_stmt'], [pytree.Leaf(token.NAME, 'foo')]) - return pytree.Node(_GRAMMAR_SYMBOL2NUMBER['suite'], [lpar1, lpar2, - simple_stmt]) + return pytree.Node(_GRAMMAR_SYMBOL2NUMBER['suite'], + [lpar1, lpar2, simple_stmt]) def _MakeNewNodeRPAR(self): return pytree.Leaf(token.RPAR, ')') @@ -147,7 +145,7 @@ def testInsertNodesWhichHasParent(self): self._simple_tree.children[0]) -class AnnotationsTest(unittest.TestCase): +class AnnotationsTest(yapf_test_helper.YAPFTest): def setUp(self): self._leaf = pytree.Leaf(token.LPAR, '(') diff --git a/yapftests/pytree_visitor_test.py b/yapftests/pytree_visitor_test.py index d0ab83457..1d908d4d3 100644 --- a/yapftests/pytree_visitor_test.py +++ b/yapftests/pytree_visitor_test.py @@ -1,4 +1,4 @@ -# Copyright 2015-2016 Google Inc. All Rights Reserved. +# Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,10 +14,12 @@ """Tests for yapf.pytree_visitor.""" import unittest +from io import StringIO -from yapf.yapflib import py3compat -from yapf.yapflib import pytree_utils -from yapf.yapflib import pytree_visitor +from yapf.pytree import pytree_utils +from yapf.pytree import pytree_visitor + +from yapftests import yapf_test_helper class _NodeNameCollector(pytree_visitor.PyTreeVisitor): @@ -46,19 +48,19 @@ def Visit_NAME(self, leaf): self.DefaultLeafVisit(leaf) -_VISITOR_TEST_SIMPLE_CODE = r''' +_VISITOR_TEST_SIMPLE_CODE = """\ foo = bar baz = x -''' +""" -_VISITOR_TEST_NESTED_CODE = r''' +_VISITOR_TEST_NESTED_CODE = """\ if x: if y: return z -''' +""" -class PytreeVisitorTest(unittest.TestCase): +class PytreeVisitorTest(yapf_test_helper.YAPFTest): def testCollectAllNodeNamesSimpleCode(self): tree = pytree_utils.ParseCodeToTree(_VISITOR_TEST_SIMPLE_CODE) @@ -68,7 +70,8 @@ def testCollectAllNodeNamesSimpleCode(self): 'file_input', 'simple_stmt', 'expr_stmt', 'NAME', 'EQUAL', 'NAME', 'NEWLINE', 'simple_stmt', 'expr_stmt', 'NAME', 'EQUAL', 'NAME', 'NEWLINE', - 'ENDMARKER'] # yapf: disable + 'ENDMARKER', + ] # yapf: disable self.assertEqual(expected_names, collector.all_node_names) expected_name_node_values = ['foo', 'bar', 'baz', 'x'] @@ -84,7 +87,8 @@ def testCollectAllNodeNamesNestedCode(self): 'suite', 'NEWLINE', 'INDENT', 'if_stmt', 'NAME', 'NAME', 'COLON', 'suite', 'NEWLINE', 'INDENT', 'simple_stmt', 'return_stmt', 'NAME', 'NAME', 'NEWLINE', - 'DEDENT', 'DEDENT', 'ENDMARKER'] # yapf: disable + 'DEDENT', 'DEDENT', 'ENDMARKER', + ] # yapf: disable self.assertEqual(expected_names, collector.all_node_names) expected_name_node_values = ['if', 'x', 'if', 'y', 'return', 'z'] @@ -94,7 +98,7 @@ def testDumper(self): # PyTreeDumper is mainly a debugging utility, so only do basic sanity # checking. tree = pytree_utils.ParseCodeToTree(_VISITOR_TEST_SIMPLE_CODE) - stream = py3compat.StringIO() + stream = StringIO() pytree_visitor.PyTreeDumper(target_stream=stream).Visit(tree) dump_output = stream.getvalue() @@ -105,7 +109,7 @@ def testDumper(self): def testDumpPyTree(self): # Similar sanity checking for the convenience wrapper DumpPyTree tree = pytree_utils.ParseCodeToTree(_VISITOR_TEST_SIMPLE_CODE) - stream = py3compat.StringIO() + stream = StringIO() pytree_visitor.DumpPyTree(tree, target_stream=stream) dump_output = stream.getvalue() diff --git a/yapftests/reformatter_basic_test.py b/yapftests/reformatter_basic_test.py new file mode 100644 index 000000000..9ef9b92ae --- /dev/null +++ b/yapftests/reformatter_basic_test.py @@ -0,0 +1,3565 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Basic tests for yapf.reformatter.""" + +import sys +import textwrap +import unittest + +from yapf.yapflib import reformatter +from yapf.yapflib import style + +from yapftests import yapf_test_helper + + +class BasicReformatterTest(yapf_test_helper.YAPFTest): + + @classmethod + def setUpClass(cls): + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testSplittingAllArgs(self): + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{split_all_comma_separated_values: true, column_limit: 40}')) + unformatted_code = textwrap.dedent("""\ + responseDict = {"timestamp": timestamp, "someValue": value, "whatever": 120} + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + responseDict = { + "timestamp": timestamp, + "someValue": value, + "whatever": 120 + } + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + unformatted_code = textwrap.dedent("""\ + yes = { 'yes': 'no', 'no': 'yes', } + """) + expected_formatted_code = textwrap.dedent("""\ + yes = { + 'yes': 'no', + 'no': 'yes', + } + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + unformatted_code = textwrap.dedent("""\ + def foo(long_arg, really_long_arg, really_really_long_arg, cant_keep_all_these_args): + pass + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + def foo(long_arg, + really_long_arg, + really_really_long_arg, + cant_keep_all_these_args): + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + unformatted_code = textwrap.dedent("""\ + foo_tuple = [long_arg, really_long_arg, really_really_long_arg, cant_keep_all_these_args] + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + foo_tuple = [ + long_arg, + really_long_arg, + really_really_long_arg, + cant_keep_all_these_args + ] + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + unformatted_code = textwrap.dedent("""\ + foo_tuple = [short, arg] + """) + expected_formatted_code = textwrap.dedent("""\ + foo_tuple = [short, arg] + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + unformatted_code = textwrap.dedent("""\ + values = [ lambda arg1, arg2: arg1 + arg2 ] + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + values = [ + lambda arg1, arg2: arg1 + arg2 + ] + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + unformatted_code = textwrap.dedent("""\ + values = [ + (some_arg1, some_arg2) for some_arg1, some_arg2 in values + ] + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + values = [ + (some_arg1, + some_arg2) + for some_arg1, some_arg2 in values + ] + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + # There is a test for split_all_top_level_comma_separated_values, with + # different expected value + unformatted_code = textwrap.dedent("""\ + someLongFunction(this_is_a_very_long_parameter, + abc=(a, this_will_just_fit_xxxxxxx)) + """) + expected_formatted_code = textwrap.dedent("""\ + someLongFunction( + this_is_a_very_long_parameter, + abc=(a, + this_will_just_fit_xxxxxxx)) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testSplittingTopLevelAllArgs(self): + style_dict = style.CreateStyleFromConfig( + '{split_all_top_level_comma_separated_values: true, ' + 'column_limit: 40}') + style.SetGlobalStyle(style_dict) + # Works the same way as split_all_comma_separated_values + unformatted_code = textwrap.dedent("""\ + responseDict = {"timestamp": timestamp, "someValue": value, "whatever": 120} + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + responseDict = { + "timestamp": timestamp, + "someValue": value, + "whatever": 120 + } + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + # Works the same way as split_all_comma_separated_values + unformatted_code = textwrap.dedent("""\ + def foo(long_arg, really_long_arg, really_really_long_arg, cant_keep_all_these_args): + pass + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + def foo(long_arg, + really_long_arg, + really_really_long_arg, + cant_keep_all_these_args): + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + # Works the same way as split_all_comma_separated_values + unformatted_code = textwrap.dedent("""\ + foo_tuple = [long_arg, really_long_arg, really_really_long_arg, cant_keep_all_these_args] + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + foo_tuple = [ + long_arg, + really_long_arg, + really_really_long_arg, + cant_keep_all_these_args + ] + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + # Works the same way as split_all_comma_separated_values + unformatted_code = textwrap.dedent("""\ + foo_tuple = [short, arg] + """) + expected_formatted_code = textwrap.dedent("""\ + foo_tuple = [short, arg] + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + # Works the same way as split_all_comma_separated_values + unformatted_code = textwrap.dedent("""\ + values = [ lambda arg1, arg2: arg1 + arg2 ] + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + values = [ + lambda arg1, arg2: arg1 + arg2 + ] + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + # There is a test for split_all_comma_separated_values, with different + # expected value + unformatted_code = textwrap.dedent("""\ + values = [ + (some_arg1, some_arg2) for some_arg1, some_arg2 in values + ] + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + values = [ + (some_arg1, some_arg2) + for some_arg1, some_arg2 in values + ] + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + # There is a test for split_all_comma_separated_values, with different + # expected value + unformatted_code = textwrap.dedent("""\ + someLongFunction(this_is_a_very_long_parameter, + abc=(a, this_will_just_fit_xxxxxxx)) + """) + expected_formatted_code = textwrap.dedent("""\ + someLongFunction( + this_is_a_very_long_parameter, + abc=(a, this_will_just_fit_xxxxxxx)) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + actual_formatted_code = reformatter.Reformat(llines) + self.assertEqual(40, len(actual_formatted_code.splitlines()[-1])) + self.assertCodeEqual(expected_formatted_code, actual_formatted_code) + + unformatted_code = textwrap.dedent("""\ + someLongFunction(this_is_a_very_long_parameter, + abc=(a, this_will_not_fit_xxxxxxxxx)) + """) + expected_formatted_code = textwrap.dedent("""\ + someLongFunction( + this_is_a_very_long_parameter, + abc=(a, + this_will_not_fit_xxxxxxxxx)) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + # This tests when there is an embedded dictionary that will fit in a line + original_multiline = style_dict['FORCE_MULTILINE_DICT'] + style_dict['FORCE_MULTILINE_DICT'] = False + style.SetGlobalStyle(style_dict) + unformatted_code = textwrap.dedent("""\ + someLongFunction(this_is_a_very_long_parameter, + abc={a: b, b: c}) + """) + expected_formatted_code = textwrap.dedent("""\ + someLongFunction( + this_is_a_very_long_parameter, + abc={ + a: b, b: c + }) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + actual_formatted_code = reformatter.Reformat(llines) + self.assertCodeEqual(expected_formatted_code, actual_formatted_code) + + # This tests when there is an embedded dictionary that will fit in a line, + # but FORCE_MULTILINE_DICT is set + style_dict['FORCE_MULTILINE_DICT'] = True + style.SetGlobalStyle(style_dict) + unformatted_code = textwrap.dedent("""\ + someLongFunction(this_is_a_very_long_parameter, + abc={a: b, b: c}) + """) + expected_formatted_code = textwrap.dedent("""\ + someLongFunction( + this_is_a_very_long_parameter, + abc={ + a: b, + b: c + }) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + actual_formatted_code = reformatter.Reformat(llines) + self.assertCodeEqual(expected_formatted_code, actual_formatted_code) + + style_dict['FORCE_MULTILINE_DICT'] = original_multiline + style.SetGlobalStyle(style_dict) + + # Exercise the case where there's no opening bracket (for a, b) + unformatted_code = textwrap.dedent("""\ + a, b = f( + a_very_long_parameter, yet_another_one, and_another) + """) + expected_formatted_code = textwrap.dedent("""\ + a, b = f( + a_very_long_parameter, yet_another_one, and_another) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + # Don't require splitting before comments. + unformatted_code = textwrap.dedent("""\ + KO = { + 'ABC': Abc, # abc + 'DEF': Def, # def + 'LOL': Lol, # wtf + 'GHI': Ghi, + 'JKL': Jkl, + } + """) + expected_formatted_code = textwrap.dedent("""\ + KO = { + 'ABC': Abc, # abc + 'DEF': Def, # def + 'LOL': Lol, # wtf + 'GHI': Ghi, + 'JKL': Jkl, + } + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testSimpleFunctionsWithTrailingComments(self): + unformatted_code = textwrap.dedent("""\ + def g(): # Trailing comment + if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and + xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): + pass + + def f( # Intermediate comment + ): + if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and + xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + def g(): # Trailing comment + if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and + xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): + pass + + + def f( # Intermediate comment + ): + if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and + xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testParamListWithTrailingComments(self): + unformatted_code = textwrap.dedent("""\ + def f(a, + b, # + c): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + def f(a, b, # + c): + pass + """) + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{based_on_style: yapf,' + ' disable_split_list_with_comment: True}')) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testBlankLinesBetweenTopLevelImportsAndVariables(self): + unformatted_code = textwrap.dedent("""\ + import foo as bar + VAR = 'baz' + """) + expected_formatted_code = textwrap.dedent("""\ + import foo as bar + + VAR = 'baz' + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + unformatted_code = textwrap.dedent("""\ + import foo as bar + + VAR = 'baz' + """) + expected_formatted_code = textwrap.dedent("""\ + import foo as bar + + + VAR = 'baz' + """) + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{based_on_style: yapf, ' + 'blank_lines_between_top_level_imports_and_variables: 2}')) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + unformatted_code = textwrap.dedent("""\ + import foo as bar + # Some comment + """) + expected_formatted_code = textwrap.dedent("""\ + import foo as bar + # Some comment + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + unformatted_code = textwrap.dedent("""\ + import foo as bar + class Baz(): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + import foo as bar + + + class Baz(): + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + unformatted_code = textwrap.dedent("""\ + import foo as bar + def foobar(): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + import foo as bar + + + def foobar(): + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + unformatted_code = textwrap.dedent("""\ + def foobar(): + from foo import Bar + Bar.baz() + """) + expected_formatted_code = textwrap.dedent("""\ + def foobar(): + from foo import Bar + Bar.baz() + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testBlankLinesAtEndOfFile(self): + unformatted_code = textwrap.dedent("""\ + def foobar(): # foo + pass + + + + """) + expected_formatted_code = textwrap.dedent("""\ + def foobar(): # foo + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + unformatted_code = textwrap.dedent("""\ + x = { 'a':37,'b':42, + + 'c':927} + + """) + expected_formatted_code = textwrap.dedent("""\ + x = {'a': 37, 'b': 42, 'c': 927} + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testIndentBlankLines(self): + unformatted_code = textwrap.dedent("""\ + class foo(object): + + def foobar(self): + + pass + + def barfoo(self, x, y): # bar + + if x: + + return y + + + def bar(): + + return 0 + """) + expected_formatted_code = """\ +class foo(object):\n \n def foobar(self):\n \n pass\n \n def barfoo(self, x, y): # bar\n \n if x:\n \n return y\n\n\ndef bar():\n \n return 0 +""" # noqa + + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{based_on_style: yapf, indent_blank_lines: true}')) + + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + unformatted_code, expected_formatted_code = (expected_formatted_code, + unformatted_code) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testMultipleUgliness(self): + unformatted_code = textwrap.dedent("""\ + x = { 'a':37,'b':42, + + 'c':927} + + y = 'hello ''world' + z = 'hello '+'world' + a = 'hello {}'.format('world') + class foo ( object ): + def f (self ): + return 37*-+2 + def g(self, x,y=42): + return y + def f ( a ) : + return 37+-+a[42-x : y**3] + """) + expected_formatted_code = textwrap.dedent("""\ + x = {'a': 37, 'b': 42, 'c': 927} + + y = 'hello ' 'world' + z = 'hello ' + 'world' + a = 'hello {}'.format('world') + + + class foo(object): + + def f(self): + return 37 * -+2 + + def g(self, x, y=42): + return y + + + def f(a): + return 37 + -+a[42 - x:y**3] + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testComments(self): + unformatted_code = textwrap.dedent("""\ + class Foo(object): + pass + + # Attached comment + class Bar(object): + pass + + global_assignment = 42 + + # Comment attached to class with decorator. + # Comment attached to class with decorator. + @noop + @noop + class Baz(object): + pass + + # Intermediate comment + + class Qux(object): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + class Foo(object): + pass + + + # Attached comment + class Bar(object): + pass + + + global_assignment = 42 + + + # Comment attached to class with decorator. + # Comment attached to class with decorator. + @noop + @noop + class Baz(object): + pass + + + # Intermediate comment + + + class Qux(object): + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testSingleComment(self): + code = textwrap.dedent("""\ + # Thing 1 + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testCommentsWithTrailingSpaces(self): + unformatted_code = textwrap.dedent("""\ + # Thing 1 \n# Thing 2 \n""") + expected_formatted_code = textwrap.dedent("""\ + # Thing 1 + # Thing 2 + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testCommentsInDataLiteral(self): + code = textwrap.dedent("""\ + def f(): + return collections.OrderedDict({ + # First comment. + 'fnord': 37, + + # Second comment. + # Continuation of second comment. + 'bork': 42, + + # Ending comment. + }) + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testEndingWhitespaceAfterSimpleStatement(self): + code = textwrap.dedent("""\ + import foo as bar + # Thing 1 + # Thing 2 + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testDocstrings(self): + unformatted_code = textwrap.dedent('''\ + u"""Module-level docstring.""" + import os + class Foo(object): + + """Class-level docstring.""" + # A comment for qux. + def qux(self): + + + """Function-level docstring. + + A multiline function docstring. + """ + print('hello {}'.format('world')) + return 42 + ''') + expected_formatted_code = textwrap.dedent('''\ + u"""Module-level docstring.""" + import os + + + class Foo(object): + """Class-level docstring.""" + + # A comment for qux. + def qux(self): + """Function-level docstring. + + A multiline function docstring. + """ + print('hello {}'.format('world')) + return 42 + ''') + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testDocstringAndMultilineComment(self): + unformatted_code = textwrap.dedent('''\ + """Hello world""" + # A multiline + # comment + class bar(object): + """class docstring""" + # class multiline + # comment + def foo(self): + """Another docstring.""" + # Another multiline + # comment + pass + ''') + expected_formatted_code = textwrap.dedent('''\ + """Hello world""" + + + # A multiline + # comment + class bar(object): + """class docstring""" + + # class multiline + # comment + def foo(self): + """Another docstring.""" + # Another multiline + # comment + pass + ''') + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testMultilineDocstringAndMultilineComment(self): + unformatted_code = textwrap.dedent('''\ + """Hello world + + RIP Dennis Richie. + """ + # A multiline + # comment + class bar(object): + """class docstring + + A classy class. + """ + # class multiline + # comment + def foo(self): + """Another docstring. + + A functional function. + """ + # Another multiline + # comment + pass + ''') + expected_formatted_code = textwrap.dedent('''\ + """Hello world + + RIP Dennis Richie. + """ + + + # A multiline + # comment + class bar(object): + """class docstring + + A classy class. + """ + + # class multiline + # comment + def foo(self): + """Another docstring. + + A functional function. + """ + # Another multiline + # comment + pass + ''') + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testTupleCommaBeforeLastParen(self): + unformatted_code = textwrap.dedent("""\ + a = ( 1, ) + """) + expected_formatted_code = textwrap.dedent("""\ + a = (1,) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testNoBreakOutsideOfBracket(self): + # FIXME(morbo): How this is formatted is not correct. But it's syntactically + # correct. + unformatted_code = textwrap.dedent("""\ + def f(): + assert port >= minimum, \ +'Unexpected port %d when minimum was %d.' % (port, minimum) + """) + expected_formatted_code = textwrap.dedent("""\ + def f(): + assert port >= minimum, 'Unexpected port %d when minimum was %d.' % (port, + minimum) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testBlankLinesBeforeDecorators(self): + unformatted_code = textwrap.dedent("""\ + @foo() + class A(object): + @bar() + @baz() + def x(self): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + @foo() + class A(object): + + @bar() + @baz() + def x(self): + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testCommentBetweenDecorators(self): + unformatted_code = textwrap.dedent("""\ + @foo() + # frob + @bar + def x (self): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + @foo() + # frob + @bar + def x(self): + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testListComprehension(self): + unformatted_code = textwrap.dedent("""\ + def given(y): + [k for k in () + if k in y] + """) + expected_formatted_code = textwrap.dedent("""\ + def given(y): + [k for k in () if k in y] + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testListComprehensionPreferOneLine(self): + unformatted_code = textwrap.dedent("""\ + def given(y): + long_variable_name = [ + long_var_name + 1 + for long_var_name in () + if long_var_name == 2] + """) + expected_formatted_code = textwrap.dedent("""\ + def given(y): + long_variable_name = [ + long_var_name + 1 for long_var_name in () if long_var_name == 2 + ] + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testListComprehensionPreferOneLineOverArithmeticSplit(self): + unformatted_code = textwrap.dedent("""\ + def given(used_identifiers): + return (sum(len(identifier) + for identifier in used_identifiers) / len(used_identifiers)) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + def given(used_identifiers): + return (sum(len(identifier) for identifier in used_identifiers) / + len(used_identifiers)) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testListComprehensionPreferThreeLinesForLineWrap(self): + unformatted_code = textwrap.dedent("""\ + def given(y): + long_variable_name = [ + long_var_name + 1 + for long_var_name, number_two in () + if long_var_name == 2 and number_two == 3] + """) + expected_formatted_code = textwrap.dedent("""\ + def given(y): + long_variable_name = [ + long_var_name + 1 + for long_var_name, number_two in () + if long_var_name == 2 and number_two == 3 + ] + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testListComprehensionPreferNoBreakForTrivialExpression(self): + unformatted_code = textwrap.dedent("""\ + def given(y): + long_variable_name = [ + long_var_name + for long_var_name, number_two in () + if long_var_name == 2 and number_two == 3] + """) + expected_formatted_code = textwrap.dedent("""\ + def given(y): + long_variable_name = [ + long_var_name for long_var_name, number_two in () + if long_var_name == 2 and number_two == 3 + ] + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testOpeningAndClosingBrackets(self): + unformatted_code = textwrap.dedent("""\ + foo( (1, ) ) + foo( ( 1, 2, 3 ) ) + foo( ( 1, 2, 3, ) ) + """) + expected_formatted_code = textwrap.dedent("""\ + foo((1,)) + foo((1, 2, 3)) + foo(( + 1, + 2, + 3, + )) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testSingleLineFunctions(self): + unformatted_code = textwrap.dedent("""\ + def foo(): return 42 + """) + expected_formatted_code = textwrap.dedent("""\ + def foo(): + return 42 + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testNoQueueSeletionInMiddleOfLine(self): + # If the queue isn't properly constructed, then a token in the middle of the + # line may be selected as the one with least penalty. The tokens after that + # one are then splatted at the end of the line with no formatting. + unformatted_code = textwrap.dedent("""\ + find_symbol(node.type) + "< " + " ".join(find_pattern(n) for n in node.child) + " >" + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + find_symbol(node.type) + "< " + " ".join( + find_pattern(n) for n in node.child) + " >" + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testNoSpacesBetweenSubscriptsAndCalls(self): + unformatted_code = textwrap.dedent("""\ + aaaaaaaaaa = bbbbbbbb.ccccccccc() [42] (a, 2) + """) + expected_formatted_code = textwrap.dedent("""\ + aaaaaaaaaa = bbbbbbbb.ccccccccc()[42](a, 2) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testNoSpacesBetweenOpeningBracketAndStartingOperator(self): + # Unary operator. + unformatted_code = textwrap.dedent("""\ + aaaaaaaaaa = bbbbbbbb.ccccccccc[ -1 ]( -42 ) + """) + expected_formatted_code = textwrap.dedent("""\ + aaaaaaaaaa = bbbbbbbb.ccccccccc[-1](-42) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + # Varargs and kwargs. + unformatted_code = textwrap.dedent("""\ + aaaaaaaaaa = bbbbbbbb.ccccccccc( *varargs ) + aaaaaaaaaa = bbbbbbbb.ccccccccc( **kwargs ) + """) + expected_formatted_code = textwrap.dedent("""\ + aaaaaaaaaa = bbbbbbbb.ccccccccc(*varargs) + aaaaaaaaaa = bbbbbbbb.ccccccccc(**kwargs) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testMultilineCommentReformatted(self): + unformatted_code = textwrap.dedent("""\ + if True: + # This is a multiline + # comment. + pass + """) + expected_formatted_code = textwrap.dedent("""\ + if True: + # This is a multiline + # comment. + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testDictionaryMakerFormatting(self): + unformatted_code = textwrap.dedent("""\ + _PYTHON_STATEMENTS = frozenset({ + lambda x, y: 'simple_stmt': 'small_stmt', 'expr_stmt': 'print_stmt', 'del_stmt': + 'pass_stmt', lambda: 'break_stmt': 'continue_stmt', 'return_stmt': 'raise_stmt', + 'yield_stmt': 'import_stmt', lambda: 'global_stmt': 'exec_stmt', 'assert_stmt': + 'if_stmt', 'while_stmt': 'for_stmt', + }) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + _PYTHON_STATEMENTS = frozenset({ + lambda x, y: 'simple_stmt': 'small_stmt', + 'expr_stmt': 'print_stmt', + 'del_stmt': 'pass_stmt', + lambda: 'break_stmt': 'continue_stmt', + 'return_stmt': 'raise_stmt', + 'yield_stmt': 'import_stmt', + lambda: 'global_stmt': 'exec_stmt', + 'assert_stmt': 'if_stmt', + 'while_stmt': 'for_stmt', + }) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testSimpleMultilineCode(self): + unformatted_code = textwrap.dedent("""\ + if True: + aaaaaaaaaaaaaa.bbbbbbbbbbbbbb.ccccccc(zzzzzzzzzzzz, \ +xxxxxxxxxxx, yyyyyyyyyyyy, vvvvvvvvv) + aaaaaaaaaaaaaa.bbbbbbbbbbbbbb.ccccccc(zzzzzzzzzzzz, \ +xxxxxxxxxxx, yyyyyyyyyyyy, vvvvvvvvv) + """) + expected_formatted_code = textwrap.dedent("""\ + if True: + aaaaaaaaaaaaaa.bbbbbbbbbbbbbb.ccccccc(zzzzzzzzzzzz, xxxxxxxxxxx, yyyyyyyyyyyy, + vvvvvvvvv) + aaaaaaaaaaaaaa.bbbbbbbbbbbbbb.ccccccc(zzzzzzzzzzzz, xxxxxxxxxxx, yyyyyyyyyyyy, + vvvvvvvvv) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testMultilineComment(self): + code = textwrap.dedent("""\ + if Foo: + # Hello world + # Yo man. + # Yo man. + # Yo man. + # Yo man. + a = 42 + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testSpaceBetweenStringAndParentheses(self): + code = textwrap.dedent("""\ + b = '0' ('hello') + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testMultilineString(self): + code = textwrap.dedent("""\ + code = textwrap.dedent('''\ + if Foo: + # Hello world + # Yo man. + # Yo man. + # Yo man. + # Yo man. + a = 42 + ''') + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + unformatted_code = textwrap.dedent('''\ + def f(): + email_text += """This is a really long docstring that goes over the column limit and is multi-line.

+ Czar: """+despot["Nicholas"]+"""
+ Minion: """+serf["Dmitri"]+"""
+ Residence: """+palace["Winter"]+"""
+ + """ + ''') # noqa + expected_formatted_code = textwrap.dedent('''\ + def f(): + email_text += """This is a really long docstring that goes over the column limit and is multi-line.

+ Czar: """ + despot["Nicholas"] + """
+ Minion: """ + serf["Dmitri"] + """
+ Residence: """ + palace["Winter"] + """
+ + """ + ''') # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testSimpleMultilineWithComments(self): + code = textwrap.dedent("""\ + if ( # This is the first comment + a and # This is the second comment + # This is the third comment + b): # A trailing comment + # Whoa! A normal comment!! + pass # Another trailing comment + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testMatchingParenSplittingMatching(self): + unformatted_code = textwrap.dedent("""\ + def f(): + raise RuntimeError('unable to find insertion point for target node', + (target,)) + """) + expected_formatted_code = textwrap.dedent("""\ + def f(): + raise RuntimeError('unable to find insertion point for target node', + (target,)) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testContinuationIndent(self): + unformatted_code = textwrap.dedent('''\ + class F: + def _ProcessArgLists(self, node): + """Common method for processing argument lists.""" + for child in node.children: + if isinstance(child, pytree.Leaf): + self._SetTokenSubtype( + child, subtype=_ARGLIST_TOKEN_TO_SUBTYPE.get( + child.value, format_token.Subtype.NONE)) + ''') + expected_formatted_code = textwrap.dedent('''\ + class F: + + def _ProcessArgLists(self, node): + """Common method for processing argument lists.""" + for child in node.children: + if isinstance(child, pytree.Leaf): + self._SetTokenSubtype( + child, + subtype=_ARGLIST_TOKEN_TO_SUBTYPE.get(child.value, + format_token.Subtype.NONE)) + ''') # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testTrailingCommaAndBracket(self): + unformatted_code = textwrap.dedent("""\ + a = { 42, } + b = ( 42, ) + c = [ 42, ] + """) + expected_formatted_code = textwrap.dedent("""\ + a = { + 42, + } + b = (42,) + c = [ + 42, + ] + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testI18n(self): + code = textwrap.dedent("""\ + N_('Some years ago - never mind how long precisely - having little or no money in my purse, and nothing particular to interest me on shore, I thought I would sail about a little and see the watery part of the world.') # A comment is here. + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + code = textwrap.dedent("""\ + foo('Fake function call') #. Some years ago - never mind how long precisely - having little or no money in my purse, and nothing particular to interest me on shore, I thought I would sail about a little and see the watery part of the world. + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testI18nCommentsInDataLiteral(self): + code = textwrap.dedent("""\ + def f(): + return collections.OrderedDict({ + #. First i18n comment. + 'bork': 'foo', + + #. Second i18n comment. + 'snork': 'bar#.*=\\\\0', + }) + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testClosingBracketIndent(self): + code = textwrap.dedent("""\ + def f(): + + def g(): + while (xxxxxxxxxxxxxxxxxxxxx(yyyyyyyyyyyyy[zzzzz]) == 'aaaaaaaaaaa' and + xxxxxxxxxxxxxxxxxxxxx( + yyyyyyyyyyyyy[zzzzz].aaaaaaaa[0]) == 'bbbbbbb'): + pass + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testClosingBracketsInlinedInCall(self): + unformatted_code = textwrap.dedent("""\ + class Foo(object): + + def bar(self): + self.aaaaaaaa = xxxxxxxxxxxxxxxxxxx.yyyyyyyyyyyyy( + self.cccccc.ddddddddd.eeeeeeee, + options={ + "forkforkfork": 1, + "borkborkbork": 2, + "corkcorkcork": 3, + "horkhorkhork": 4, + "porkporkpork": 5, + }) + """) + expected_formatted_code = textwrap.dedent("""\ + class Foo(object): + + def bar(self): + self.aaaaaaaa = xxxxxxxxxxxxxxxxxxx.yyyyyyyyyyyyy( + self.cccccc.ddddddddd.eeeeeeee, + options={ + "forkforkfork": 1, + "borkborkbork": 2, + "corkcorkcork": 3, + "horkhorkhork": 4, + "porkporkpork": 5, + }) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testLineWrapInForExpression(self): + code = textwrap.dedent("""\ + class A: + + def x(self, node, name, n=1): + for i, child in enumerate( + itertools.ifilter(lambda c: pytree_utils.NodeName(c) == name, + node.pre_order())): + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testFunctionCallContinuationLine(self): + code = textwrap.dedent("""\ + class foo: + + def bar(self, node, name, n=1): + if True: + if True: + return [(aaaaaaaaaa, + bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb( + cccc, ddddddddddddddddddddddddddddddddddddd))] + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testI18nNonFormatting(self): + code = textwrap.dedent("""\ + class F(object): + + def __init__(self, fieldname, + #. Error message indicating an invalid e-mail address. + message=N_('Please check your email address.'), **kwargs): + pass + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testNoSpaceBetweenUnaryOpAndOpeningParen(self): + code = textwrap.dedent("""\ + if ~(a or b): + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testCommentBeforeFuncDef(self): + code = textwrap.dedent("""\ + class Foo(object): + + a = 42 + + # This is a comment. + def __init__(self, + xxxxxxx, + yyyyy=0, + zzzzzzz=None, + aaaaaaaaaaaaaaaaaa=False, + bbbbbbbbbbbbbbb=False): + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testExcessLineCountWithDefaultKeywords(self): + unformatted_code = textwrap.dedent("""\ + class Fnord(object): + def Moo(self): + aaaaaaaaaaaaaaaa = self._bbbbbbbbbbbbbbbbbbbbbbb( + ccccccccccccc=ccccccccccccc, ddddddd=ddddddd, eeee=eeee, + fffff=fffff, ggggggg=ggggggg, hhhhhhhhhhhhh=hhhhhhhhhhhhh, + iiiiiii=iiiiiiiiiiiiii) + """) + expected_formatted_code = textwrap.dedent("""\ + class Fnord(object): + + def Moo(self): + aaaaaaaaaaaaaaaa = self._bbbbbbbbbbbbbbbbbbbbbbb( + ccccccccccccc=ccccccccccccc, + ddddddd=ddddddd, + eeee=eeee, + fffff=fffff, + ggggggg=ggggggg, + hhhhhhhhhhhhh=hhhhhhhhhhhhh, + iiiiiii=iiiiiiiiiiiiii) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testSpaceAfterNotOperator(self): + code = textwrap.dedent("""\ + if not (this and that): + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testNoPenaltySplitting(self): + code = textwrap.dedent("""\ + def f(): + if True: + if True: + python_files.extend( + os.path.join(filename, f) + for f in os.listdir(filename) + if IsPythonFile(os.path.join(filename, f))) + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testExpressionPenalties(self): + code = textwrap.dedent("""\ + def f(): + if ((left.value == '(' and right.value == ')') or + (left.value == '[' and right.value == ']') or + (left.value == '{' and right.value == '}')): + return False + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testLineDepthOfSingleLineStatement(self): + unformatted_code = textwrap.dedent("""\ + while True: continue + for x in range(3): continue + try: a = 42 + except: b = 42 + with open(a) as fd: a = fd.read() + """) + expected_formatted_code = textwrap.dedent("""\ + while True: + continue + for x in range(3): + continue + try: + a = 42 + except: + b = 42 + with open(a) as fd: + a = fd.read() + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testSplitListWithTerminatingComma(self): + unformatted_code = textwrap.dedent("""\ + FOO = ['bar', 'baz', 'mux', 'qux', 'quux', 'quuux', 'quuuux', + 'quuuuux', 'quuuuuux', 'quuuuuuux', lambda a, b: 37,] + """) + expected_formatted_code = textwrap.dedent("""\ + FOO = [ + 'bar', + 'baz', + 'mux', + 'qux', + 'quux', + 'quuux', + 'quuuux', + 'quuuuux', + 'quuuuuux', + 'quuuuuuux', + lambda a, b: 37, + ] + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testSplitListWithInterspersedComments(self): + code = textwrap.dedent("""\ + FOO = [ + 'bar', # bar + 'baz', # baz + 'mux', # mux + 'qux', # qux + 'quux', # quux + 'quuux', # quuux + 'quuuux', # quuuux + 'quuuuux', # quuuuux + 'quuuuuux', # quuuuuux + 'quuuuuuux', # quuuuuuux + lambda a, b: 37 # lambda + ] + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testRelativeImportStatements(self): + code = textwrap.dedent("""\ + from ... import bork + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testSingleLineList(self): + # A list on a single line should prefer to remain contiguous. + unformatted_code = textwrap.dedent("""\ + bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb = aaaaaaaaaaa( + ("...", "."), "..", + ".............................................." + ) + """) + expected_formatted_code = textwrap.dedent("""\ + bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb = aaaaaaaaaaa( + ("...", "."), "..", "..............................................") + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testBlankLinesBeforeFunctionsNotInColumnZero(self): + unformatted_code = textwrap.dedent("""\ + import signal + + + try: + signal.SIGALRM + # .................................................................. + # ............................................................... + + + def timeout(seconds=1): + pass + except: + pass + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + import signal + + try: + signal.SIGALRM + + # .................................................................. + # ............................................................... + + + def timeout(seconds=1): + pass + except: + pass + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testNoKeywordArgumentBreakage(self): + code = textwrap.dedent("""\ + class A(object): + + def b(self): + if self.aaaaaaaaaaaaaaaaaaaa not in self.bbbbbbbbbb( + cccccccccccccccccccc=True): + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testTrailerOnSingleLine(self): + code = textwrap.dedent("""\ + urlpatterns = patterns('', url(r'^$', 'homepage_view'), + url(r'^/login/$', 'login_view'), + url(r'^/login/$', 'logout_view'), + url(r'^/user/(?P\\w+)/$', 'profile_view')) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testIfConditionalParens(self): + code = textwrap.dedent("""\ + class Foo: + + def bar(): + if True: + if (child.type == grammar_token.NAME and + child.value in substatement_names): + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testContinuationMarkers(self): + code = textwrap.dedent("""\ + text = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec a diam lectus. "\\ + "Sed sit amet ipsum mauris. Maecenas congue ligula ac quam viverra nec consectetur "\\ + "ante hendrerit. Donec et mollis dolor. Praesent et diam eget libero egestas mattis "\\ + "sit amet vitae augue. Nam tincidunt congue enim, ut porta lorem lacinia consectetur. "\\ + "Donec ut libero sed arcu vehicula ultricies a non tortor. Lorem ipsum dolor sit amet" + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + code = textwrap.dedent("""\ + from __future__ import nested_scopes, generators, division, absolute_import, with_statement, \\ + print_function, unicode_literals + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + code = textwrap.dedent("""\ + if aaaaaaaaa == 42 and bbbbbbbbbbbbbb == 42 and \\ + cccccccc == 42: + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testCommentsWithContinuationMarkers(self): + code = textwrap.dedent("""\ + def fn(arg): + v = fn2(key1=True, + #c1 + key2=arg)\\ + .fn3() + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testMultipleContinuationMarkers(self): + code = textwrap.dedent("""\ + xyz = \\ + \\ + some_thing() + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testContinuationMarkerAfterStringWithContinuation(self): + code = textwrap.dedent("""\ + s = 'foo \\ + bar' \\ + .format() + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testEmptyContainers(self): + code = textwrap.dedent("""\ + flags.DEFINE_list( + 'output_dirs', [], + 'Lorem ipsum dolor sit amet, consetetur adipiscing elit. Donec a diam lectus. ' + 'Sed sit amet ipsum mauris. Maecenas congue.') + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testSplitStringsIfSurroundedByParens(self): + unformatted_code = textwrap.dedent("""\ + a = foo.bar({'xxxxxxxxxxxxxxxxxxxxxxx' 'yyyyyyyyyyyyyyyyyyyyyyyyyy': baz[42]} + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' 'bbbbbbbbbbbbbbbbbbbbbbbbbb' 'cccccccccccccccccccccccccccccccc' 'ddddddddddddddddddddddddddddd') + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + a = foo.bar({'xxxxxxxxxxxxxxxxxxxxxxx' + 'yyyyyyyyyyyyyyyyyyyyyyyyyy': baz[42]} + + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + 'bbbbbbbbbbbbbbbbbbbbbbbbbb' + 'cccccccccccccccccccccccccccccccc' + 'ddddddddddddddddddddddddddddd') + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + code = textwrap.dedent("""\ + a = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' \ +'bbbbbbbbbbbbbbbbbbbbbbbbbb' 'cccccccccccccccccccccccccccccccc' \ +'ddddddddddddddddddddddddddddd' + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testMultilineShebang(self): + code = textwrap.dedent("""\ + #!/bin/sh + if "true" : '''\' + then + + export FOO=123 + exec /usr/bin/env python "$0" "$@" + + exit 127 + fi + ''' + + import os + + assert os.environ['FOO'] == '123' + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testNoSplittingAroundTermOperators(self): + code = textwrap.dedent("""\ + a_very_long_function_call_yada_yada_etc_etc_etc(long_arg1, + long_arg2 / long_arg3) + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testNoSplittingAroundCompOperators(self): + code = textwrap.dedent("""\ + c = (aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa is not bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb) + c = (aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa in bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb) + c = (aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa not in bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb) + + c = (aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa is bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb) + c = (aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa <= bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb) + """) # noqa + expected_code = textwrap.dedent("""\ + c = ( + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + is not bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb) + c = ( + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + in bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb) + c = ( + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + not in bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb) + + c = ( + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + is bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb) + c = ( + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + <= bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb) + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(expected_code, reformatter.Reformat(llines)) + + def testNoSplittingWithinSubscriptList(self): + code = textwrap.dedent("""\ + somequitelongvariablename.somemember[(a, b)] = { + 'somelongkey': 1, + 'someotherlongkey': 2 + } + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testExcessCharacters(self): + code = textwrap.dedent("""\ + class foo: + + def bar(self): + self.write(s=[ + '%s%s %s' % ('many of really', 'long strings', '+ just makes up 81') + ]) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + unformatted_code = textwrap.dedent("""\ + def _(): + if True: + if True: + if contract == allow_contract and attr_dict.get(if_attribute) == has_value: + return True + """) # noqa + expected_code = textwrap.dedent("""\ + def _(): + if True: + if True: + if contract == allow_contract and attr_dict.get( + if_attribute) == has_value: + return True + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_code, reformatter.Reformat(llines)) + + def testDictSetGenerator(self): + code = textwrap.dedent("""\ + foo = { + variable: 'hello world. How are you today?' + for variable in fnord + if variable != 37 + } + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + unformatted_code = textwrap.dedent("""\ + foo = { + x: x + for x in fnord + } + """) # noqa + expected_code = textwrap.dedent("""\ + foo = {x: x for x in fnord} + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_code, reformatter.Reformat(llines)) + + def testUnaryOpInDictionaryValue(self): + code = textwrap.dedent("""\ + beta = "123" + + test = {'alpha': beta[-1]} + + print(beta[-1]) + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testUnaryNotOperator(self): + code = textwrap.dedent("""\ + if True: + if True: + if True: + if True: + remote_checksum = self.get_checksum(conn, tmp, dest, inject, + not directory_prepended, source) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testRelaxArraySubscriptAffinity(self): + code = textwrap.dedent("""\ + class A(object): + + def f(self, aaaaaaaaa, bbbbbbbbbbbbb, row): + if True: + if True: + if True: + if True: + if row[4] is None or row[5] is None: + bbbbbbbbbbbbb[ + '..............'] = row[5] if row[5] is not None else 5 + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testFunctionCallInDict(self): + code = textwrap.dedent("""\ + a = {'a': b(c=d, **e)} + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testFunctionCallInNestedDict(self): + code = textwrap.dedent("""\ + a = {'a': {'a': {'a': b(c=d, **e)}}} + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testUnbreakableNot(self): + code = textwrap.dedent("""\ + def test(): + if not "Foooooooooooooooooooooooooooooo" or "Foooooooooooooooooooooooooooooo" == "Foooooooooooooooooooooooooooooo": + pass + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testSplitListWithComment(self): + code = textwrap.dedent("""\ + a = [ + 'a', + 'b', + 'c' # hello world + ] + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testOverColumnLimit(self): + unformatted_code = textwrap.dedent("""\ + class Test: + + def testSomething(self): + expected = { + ('aaaaaaaaaaaaa', 'bbbb'): 'ccccccccccccccccccccccccccccccccccccccccccc', + ('aaaaaaaaaaaaa', 'bbbb'): 'ccccccccccccccccccccccccccccccccccccccccccc', + ('aaaaaaaaaaaaa', 'bbbb'): 'ccccccccccccccccccccccccccccccccccccccccccc', + } + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + class Test: + + def testSomething(self): + expected = { + ('aaaaaaaaaaaaa', 'bbbb'): + 'ccccccccccccccccccccccccccccccccccccccccccc', + ('aaaaaaaaaaaaa', 'bbbb'): + 'ccccccccccccccccccccccccccccccccccccccccccc', + ('aaaaaaaaaaaaa', 'bbbb'): + 'ccccccccccccccccccccccccccccccccccccccccccc', + } + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testEndingComment(self): + code = textwrap.dedent("""\ + a = f( + a="something", + b="something requiring comment which is quite long", # comment about b (pushes line over 79) + c="something else, about which comment doesn't make sense") + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testContinuationSpaceRetention(self): + code = textwrap.dedent("""\ + def fn(): + return module \\ + .method(Object(data, + fn2(arg) + )) + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testIfExpressionWithFunctionCall(self): + code = textwrap.dedent("""\ + if x or z.y( + a, + c, + aaaaaaaaaaaaaaaaaaaaa=aaaaaaaaaaaaaaaaaa, + bbbbbbbbbbbbbbbbbbbbb=bbbbbbbbbbbbbbbbbb): + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testUnformattedAfterMultilineString(self): + code = textwrap.dedent("""\ + def foo(): + com_text = \\ + ''' + TEST + ''' % (input_fname, output_fname) + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testNoSpacesAroundKeywordDefaultValues(self): + code = textwrap.dedent("""\ + sources = { + 'json': request.get_json(silent=True) or {}, + 'json2': request.get_json(silent=True), + } + json = request.get_json(silent=True) or {} + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testNoSplittingBeforeEndingSubscriptBracket(self): + unformatted_code = textwrap.dedent("""\ + if True: + if True: + status = cf.describe_stacks(StackName=stackname)[u'Stacks'][0][u'StackStatus'] + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + if True: + if True: + status = cf.describe_stacks( + StackName=stackname)[u'Stacks'][0][u'StackStatus'] + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testNoSplittingOnSingleArgument(self): + unformatted_code = textwrap.dedent("""\ + xxxxxxxxxxxxxx = (re.search(r'(\\d+\\.\\d+\\.\\d+\\.)\\d+', + aaaaaaa.bbbbbbbbbbbb).group(1) + + re.search(r'\\d+\\.\\d+\\.\\d+\\.(\\d+)', + ccccccc).group(1)) + xxxxxxxxxxxxxx = (re.search(r'(\\d+\\.\\d+\\.\\d+\\.)\\d+', + aaaaaaa.bbbbbbbbbbbb).group(a.b) + + re.search(r'\\d+\\.\\d+\\.\\d+\\.(\\d+)', + ccccccc).group(c.d)) + """) + expected_formatted_code = textwrap.dedent("""\ + xxxxxxxxxxxxxx = ( + re.search(r'(\\d+\\.\\d+\\.\\d+\\.)\\d+', aaaaaaa.bbbbbbbbbbbb).group(1) + + re.search(r'\\d+\\.\\d+\\.\\d+\\.(\\d+)', ccccccc).group(1)) + xxxxxxxxxxxxxx = ( + re.search(r'(\\d+\\.\\d+\\.\\d+\\.)\\d+', aaaaaaa.bbbbbbbbbbbb).group(a.b) + + re.search(r'\\d+\\.\\d+\\.\\d+\\.(\\d+)', ccccccc).group(c.d)) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testSplittingArraysSensibly(self): + unformatted_code = textwrap.dedent("""\ + while True: + while True: + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa = list['bbbbbbbbbbbbbbbbbbbbbbbbb'].split(',') + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa = list('bbbbbbbbbbbbbbbbbbbbbbbbb').split(',') + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + while True: + while True: + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa = list[ + 'bbbbbbbbbbbbbbbbbbbbbbbbb'].split(',') + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa = list( + 'bbbbbbbbbbbbbbbbbbbbbbbbb').split(',') + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testComprehensionForAndIf(self): + unformatted_code = textwrap.dedent("""\ + class f: + + def __repr__(self): + tokens_repr = ','.join(['{0}({1!r})'.format(tok.name, tok.value) for tok in self._tokens]) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + class f: + + def __repr__(self): + tokens_repr = ','.join( + ['{0}({1!r})'.format(tok.name, tok.value) for tok in self._tokens]) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testFunctionCallArguments(self): + unformatted_code = textwrap.dedent("""\ + def f(): + if True: + pytree_utils.InsertNodesBefore(_CreateCommentsFromPrefix( + comment_prefix, comment_lineno, comment_column, + standalone=True), ancestor_at_indent) + pytree_utils.InsertNodesBefore(_CreateCommentsFromPrefix( + comment_prefix, comment_lineno, comment_column, + standalone=True)) + """) + expected_formatted_code = textwrap.dedent("""\ + def f(): + if True: + pytree_utils.InsertNodesBefore( + _CreateCommentsFromPrefix( + comment_prefix, comment_lineno, comment_column, standalone=True), + ancestor_at_indent) + pytree_utils.InsertNodesBefore( + _CreateCommentsFromPrefix( + comment_prefix, comment_lineno, comment_column, standalone=True)) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testBinaryOperators(self): + unformatted_code = textwrap.dedent("""\ + a = b ** 37 + c = (20 ** -3) / (_GRID_ROWS ** (code_length - 10)) + """) + expected_formatted_code = textwrap.dedent("""\ + a = b**37 + c = (20**-3) / (_GRID_ROWS**(code_length - 10)) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + code = textwrap.dedent("""\ + def f(): + if True: + if (self.stack[-1].split_before_closing_bracket and + # FIXME(morbo): Use the 'matching_bracket' instead of this. + # FIXME(morbo): Don't forget about tuples! + current.value in ']}'): + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testContiguousList(self): + code = textwrap.dedent("""\ + [retval1, retval2] = a_very_long_function(argument_1, argument2, argument_3, + argument_4) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testArgsAndKwargsFormatting(self): + code = textwrap.dedent("""\ + a(a=aaaaaaaaaaaaaaaaaaaaa, + b=aaaaaaaaaaaaaaaaaaaaaaaa, + c=aaaaaaaaaaaaaaaaaa, + *d, + **e) + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + code = textwrap.dedent("""\ + def foo(): + return [ + Bar(xxx='some string', + yyy='another long string', + zzz='a third long string') + ] + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testCommentColumnLimitOverflow(self): + code = textwrap.dedent("""\ + def f(): + if True: + TaskManager.get_tags = MagicMock( + name='get_tags_mock', + return_value=[157031694470475], + # side_effect=[(157031694470475), (157031694470475),], + ) + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testMultilineLambdas(self): + unformatted_code = textwrap.dedent("""\ + class SomeClass(object): + do_something = True + + def succeeded(self, dddddddddddddd): + d = defer.succeed(None) + + if self.do_something: + d.addCallback(lambda _: self.aaaaaa.bbbbbbbbbbbbbbbb.cccccccccccccccccccccccccccccccc(dddddddddddddd)) + return d + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + class SomeClass(object): + do_something = True + + def succeeded(self, dddddddddddddd): + d = defer.succeed(None) + + if self.do_something: + d.addCallback(lambda _: self.aaaaaa.bbbbbbbbbbbbbbbb. + cccccccccccccccccccccccccccccccc(dddddddddddddd)) + return d + """) + + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{based_on_style: yapf, allow_multiline_lambdas: true}')) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testMultilineDictionaryKeys(self): + unformatted_code = textwrap.dedent("""\ + MAP_WITH_LONG_KEYS = { + ('lorem ipsum', 'dolor sit amet'): + 1, + ('consectetur adipiscing elit.', 'Vestibulum mauris justo, ornare eget dolor eget'): + 2, + ('vehicula convallis nulla. Vestibulum dictum nisl in malesuada finibus.',): + 3 + } + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + MAP_WITH_LONG_KEYS = { + ('lorem ipsum', 'dolor sit amet'): + 1, + ('consectetur adipiscing elit.', + 'Vestibulum mauris justo, ornare eget dolor eget'): + 2, + ('vehicula convallis nulla. Vestibulum dictum nisl in malesuada finibus.',): + 3 + } + """) # noqa + + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig('{based_on_style: yapf, ' + 'allow_multiline_dictionary_keys: true}')) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testStableDictionaryFormatting(self): + code = textwrap.dedent("""\ + class A(object): + + def method(self): + filters = { + 'expressions': [{ + 'field': { + 'search_field': { + 'user_field': 'latest_party__number_of_guests' + }, + } + }] + } + """) # noqa + + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig('{based_on_style: pep8, indent_width: 2, ' + 'continuation_indent_width: 4, ' + 'indent_dictionary_value: True}')) + + llines = yapf_test_helper.ParseAndUnwrap(code) + reformatted_code = reformatter.Reformat(llines) + self.assertCodeEqual(code, reformatted_code) + + llines = yapf_test_helper.ParseAndUnwrap(reformatted_code) + reformatted_code = reformatter.Reformat(llines) + self.assertCodeEqual(code, reformatted_code) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testStableInlinedDictionaryFormatting(self): + try: + style.SetGlobalStyle(style.CreatePEP8Style()) + unformatted_code = textwrap.dedent("""\ + def _(): + url = "http://{0}/axis-cgi/admin/param.cgi?{1}".format( + value, urllib.urlencode({'action': 'update', 'parameter': value})) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + def _(): + url = "http://{0}/axis-cgi/admin/param.cgi?{1}".format( + value, urllib.urlencode({ + 'action': 'update', + 'parameter': value + })) + """) + + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + reformatted_code = reformatter.Reformat(llines) + self.assertCodeEqual(expected_formatted_code, reformatted_code) + + llines = yapf_test_helper.ParseAndUnwrap(reformatted_code) + reformatted_code = reformatter.Reformat(llines) + self.assertCodeEqual(expected_formatted_code, reformatted_code) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testDontSplitKeywordValueArguments(self): + unformatted_code = textwrap.dedent("""\ + def mark_game_scored(gid): + _connect.execute(_games.update().where(_games.c.gid == gid).values( + scored=True)) + """) + expected_formatted_code = textwrap.dedent("""\ + def mark_game_scored(gid): + _connect.execute( + _games.update().where(_games.c.gid == gid).values(scored=True)) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testDontAddBlankLineAfterMultilineString(self): + code = textwrap.dedent("""\ + query = '''SELECT id + FROM table + WHERE day in {}''' + days = ",".join(days) + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testFormattingListComprehensions(self): + code = textwrap.dedent("""\ + def a(): + if True: + if True: + if True: + columns = [ + x for x, y in self._heap_this_is_very_long if x.route[0] == choice + ] + self._heap = [x for x in self._heap if x.route and x.route[0] == choice] + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testNoSplittingWhenBinPacking(self): + code = textwrap.dedent("""\ + a_very_long_function_name( + long_argument_name_1=1, + long_argument_name_2=2, + long_argument_name_3=3, + long_argument_name_4=4, + ) + + a_very_long_function_name( + long_argument_name_1=1, long_argument_name_2=2, long_argument_name_3=3, + long_argument_name_4=4 + ) + """) # noqa + + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{based_on_style: pep8, indent_width: 2, ' + 'continuation_indent_width: 4, indent_dictionary_value: True, ' + 'dedent_closing_brackets: True, ' + 'split_before_named_assigns: False}')) + + llines = yapf_test_helper.ParseAndUnwrap(code) + reformatted_code = reformatter.Reformat(llines) + self.assertCodeEqual(code, reformatted_code) + + llines = yapf_test_helper.ParseAndUnwrap(reformatted_code) + reformatted_code = reformatter.Reformat(llines) + self.assertCodeEqual(code, reformatted_code) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testNotSplittingAfterSubscript(self): + unformatted_code = textwrap.dedent("""\ + if not aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.b(c == d[ + 'eeeeee']).ffffff(): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + if not aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.b( + c == d['eeeeee']).ffffff(): + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testSplittingOneArgumentList(self): + unformatted_code = textwrap.dedent("""\ + def _(): + if True: + if True: + if True: + if True: + if True: + boxes[id_] = np.concatenate((points.min(axis=0), qoints.max(axis=0))) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + def _(): + if True: + if True: + if True: + if True: + if True: + boxes[id_] = np.concatenate( + (points.min(axis=0), qoints.max(axis=0))) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testSplittingBeforeFirstElementListArgument(self): + unformatted_code = textwrap.dedent("""\ + class _(): + @classmethod + def _pack_results_for_constraint_or(cls, combination, constraints): + if True: + if True: + if True: + return cls._create_investigation_result( + ( + clue for clue in combination if not clue == Verifier.UNMATCHED + ), constraints, InvestigationResult.OR + ) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + class _(): + + @classmethod + def _pack_results_for_constraint_or(cls, combination, constraints): + if True: + if True: + if True: + return cls._create_investigation_result( + (clue for clue in combination if not clue == Verifier.UNMATCHED), + constraints, InvestigationResult.OR) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testSplittingArgumentsTerminatedByComma(self): + unformatted_code = textwrap.dedent("""\ + function_name(argument_name_1=1, argument_name_2=2, argument_name_3=3) + + function_name(argument_name_1=1, argument_name_2=2, argument_name_3=3,) + + a_very_long_function_name(long_argument_name_1=1, long_argument_name_2=2, long_argument_name_3=3, long_argument_name_4=4) + + a_very_long_function_name(long_argument_name_1, long_argument_name_2, long_argument_name_3, long_argument_name_4,) + + r =f0 (1, 2,3,) + + r =f0 (1,) + + r =f0 (a=1,) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + function_name(argument_name_1=1, argument_name_2=2, argument_name_3=3) + + function_name( + argument_name_1=1, + argument_name_2=2, + argument_name_3=3, + ) + + a_very_long_function_name( + long_argument_name_1=1, + long_argument_name_2=2, + long_argument_name_3=3, + long_argument_name_4=4) + + a_very_long_function_name( + long_argument_name_1, + long_argument_name_2, + long_argument_name_3, + long_argument_name_4, + ) + + r = f0( + 1, + 2, + 3, + ) + + r = f0( + 1, + ) + + r = f0( + a=1, + ) + """) + + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{based_on_style: yapf, ' + 'split_arguments_when_comma_terminated: True}')) + + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + reformatted_code = reformatter.Reformat(llines) + self.assertCodeEqual(expected_formatted_code, reformatted_code) + + llines = yapf_test_helper.ParseAndUnwrap(reformatted_code) + reformatted_code = reformatter.Reformat(llines) + self.assertCodeEqual(expected_formatted_code, reformatted_code) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testImportAsList(self): + code = textwrap.dedent("""\ + from toto import titi, tata, tutu # noqa + from toto import titi, tata, tutu + from toto import (titi, tata, tutu) + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testDictionaryValuesOnOwnLines(self): + unformatted_code = textwrap.dedent("""\ + a = { + 'aaaaaaaaaaaaaaaaaaaaaaaa': + Check('ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ', '=', True), + 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbb': + Check('YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY', '=', True), + 'ccccccccccccccc': + Check('XXXXXXXXXXXXXXXXXXX', '!=', 'SUSPENDED'), + 'dddddddddddddddddddddddddddddd': + Check('WWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWW', '=', False), + 'eeeeeeeeeeeeeeeeeeeeeeeeeeeee': + Check('VVVVVVVVVVVVVVVVVVVVVVVVVVVVVV', '=', False), + 'ffffffffffffffffffffffffff': + Check('UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU', '=', True), + 'ggggggggggggggggg': + Check('TTTTTTTTTTTTTTTTTTTTTTTTTTTTTT', '=', True), + 'hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh': + Check('SSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSS', '=', True), + 'iiiiiiiiiiiiiiiiiiiiiiii': + Check('RRRRRRRRRRRRRRRRRRRRRRRRRRR', '=', True), + 'jjjjjjjjjjjjjjjjjjjjjjjjjj': + Check('QQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQ', '=', False), + } + """) + expected_formatted_code = textwrap.dedent("""\ + a = { + 'aaaaaaaaaaaaaaaaaaaaaaaa': + Check('ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ', '=', True), + 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbb': + Check('YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY', '=', True), + 'ccccccccccccccc': + Check('XXXXXXXXXXXXXXXXXXX', '!=', 'SUSPENDED'), + 'dddddddddddddddddddddddddddddd': + Check('WWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWW', '=', False), + 'eeeeeeeeeeeeeeeeeeeeeeeeeeeee': + Check('VVVVVVVVVVVVVVVVVVVVVVVVVVVVVV', '=', False), + 'ffffffffffffffffffffffffff': + Check('UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU', '=', True), + 'ggggggggggggggggg': + Check('TTTTTTTTTTTTTTTTTTTTTTTTTTTTTT', '=', True), + 'hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh': + Check('SSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSS', '=', True), + 'iiiiiiiiiiiiiiiiiiiiiiii': + Check('RRRRRRRRRRRRRRRRRRRRRRRRRRR', '=', True), + 'jjjjjjjjjjjjjjjjjjjjjjjjjj': + Check('QQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQ', '=', False), + } + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testDictionaryOnOwnLine(self): + unformatted_code = textwrap.dedent("""\ + doc = test_utils.CreateTestDocumentViaController( + content={ 'a': 'b' }, + branch_key=branch.key, + collection_key=collection.key) + """) + expected_formatted_code = textwrap.dedent("""\ + doc = test_utils.CreateTestDocumentViaController( + content={'a': 'b'}, branch_key=branch.key, collection_key=collection.key) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + unformatted_code = textwrap.dedent("""\ + doc = test_utils.CreateTestDocumentViaController( + content={ 'a': 'b' }, + branch_key=branch.key, + collection_key=collection.key, + collection_key2=collection.key2) + """) + expected_formatted_code = textwrap.dedent("""\ + doc = test_utils.CreateTestDocumentViaController( + content={'a': 'b'}, + branch_key=branch.key, + collection_key=collection.key, + collection_key2=collection.key2) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testNestedListsInDictionary(self): + unformatted_code = textwrap.dedent("""\ + _A = { + 'cccccccccc': ('^^1',), + 'rrrrrrrrrrrrrrrrrrrrrrrrr': ('^7913', # AAAAAAAAAAAAAA. + ), + 'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee': ('^6242', # BBBBBBBBBBBBBBB. + ), + 'vvvvvvvvvvvvvvvvvvv': ('^27959', # CCCCCCCCCCCCCCCCCC. + '^19746', # DDDDDDDDDDDDDDDDDDDDDDD. + '^22907', # EEEEEEEEEEEEEEEEEEEEEEEE. + '^21098', # FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF. + '^22826', # GGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGG. + '^22769', # HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH. + '^22935', # IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII. + '^3982', # JJJJJJJJJJJJJ. + ), + 'uuuuuuuuuuuu': ('^19745', # LLLLLLLLLLLLLLLLLLLLLLLLLL. + '^21324', # MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM. + '^22831', # NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN. + '^17081', # OOOOOOOOOOOOOOOOOOOOO. + ), + 'eeeeeeeeeeeeee': ( + '^9416', # Reporter email. Not necessarily the reporter. + '^^3', # This appears to be the raw email field. + ), + 'cccccccccc': ('^21109', # PPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPP. + ), + } + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + _A = { + 'cccccccccc': ('^^1',), + 'rrrrrrrrrrrrrrrrrrrrrrrrr': ( + '^7913', # AAAAAAAAAAAAAA. + ), + 'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee': ( + '^6242', # BBBBBBBBBBBBBBB. + ), + 'vvvvvvvvvvvvvvvvvvv': ( + '^27959', # CCCCCCCCCCCCCCCCCC. + '^19746', # DDDDDDDDDDDDDDDDDDDDDDD. + '^22907', # EEEEEEEEEEEEEEEEEEEEEEEE. + '^21098', # FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF. + '^22826', # GGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGG. + '^22769', # HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH. + '^22935', # IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII. + '^3982', # JJJJJJJJJJJJJ. + ), + 'uuuuuuuuuuuu': ( + '^19745', # LLLLLLLLLLLLLLLLLLLLLLLLLL. + '^21324', # MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM. + '^22831', # NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN. + '^17081', # OOOOOOOOOOOOOOOOOOOOO. + ), + 'eeeeeeeeeeeeee': ( + '^9416', # Reporter email. Not necessarily the reporter. + '^^3', # This appears to be the raw email field. + ), + 'cccccccccc': ( + '^21109', # PPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPP. + ), + } + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testNestedDictionary(self): + unformatted_code = textwrap.dedent("""\ + class _(): + def _(): + breadcrumbs = [{'name': 'Admin', + 'url': url_for(".home")}, + {'title': title},] + breadcrumbs = [{'name': 'Admin', + 'url': url_for(".home")}, + {'title': title}] + """) + expected_formatted_code = textwrap.dedent("""\ + class _(): + def _(): + breadcrumbs = [ + { + 'name': 'Admin', + 'url': url_for(".home") + }, + { + 'title': title + }, + ] + breadcrumbs = [{'name': 'Admin', 'url': url_for(".home")}, {'title': title}] + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testDictionaryElementsOnOneLine(self): + code = textwrap.dedent("""\ + class _(): + + @mock.patch.dict( + os.environ, + {'HTTP_' + xsrf._XSRF_TOKEN_HEADER.replace('-', '_'): 'atoken'}) + def _(): + pass + + + AAAAAAAAAAAAAAAAAAAAAAAA = { + Environment.XXXXXXXXXX: 'some text more text even more tex', + Environment.YYYYYYY: 'some text more text even more text yet ag', + Environment.ZZZZZZZZZZZ: 'some text more text even more text yet again tex', + } + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testNotInParams(self): + unformatted_code = textwrap.dedent("""\ + list("a long line to break the line. a long line to break the brk a long lin", not True) + """) # noqa + expected_code = textwrap.dedent("""\ + list("a long line to break the line. a long line to break the brk a long lin", + not True) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_code, reformatter.Reformat(llines)) + + def testNamedAssignNotAtEndOfLine(self): + unformatted_code = textwrap.dedent("""\ + def _(): + if True: + with py3compat.open_with_encoding(filename, mode='w', + encoding=encoding) as fd: + pass + """) + expected_code = textwrap.dedent("""\ + def _(): + if True: + with py3compat.open_with_encoding( + filename, mode='w', encoding=encoding) as fd: + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_code, reformatter.Reformat(llines)) + + def testBlankLineBeforeClassDocstring(self): + unformatted_code = textwrap.dedent('''\ + class A: + + """Does something. + + Also, here are some details. + """ + + def __init__(self): + pass + ''') + expected_code = textwrap.dedent('''\ + class A: + """Does something. + + Also, here are some details. + """ + + def __init__(self): + pass + ''') + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_code, reformatter.Reformat(llines)) + + unformatted_code = textwrap.dedent('''\ + class A: + + """Does something. + + Also, here are some details. + """ + + def __init__(self): + pass + ''') + expected_formatted_code = textwrap.dedent('''\ + class A: + + """Does something. + + Also, here are some details. + """ + + def __init__(self): + pass + ''') + + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{based_on_style: yapf, ' + 'blank_line_before_class_docstring: True}')) + + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testBlankLineBeforeModuleDocstring(self): + unformatted_code = textwrap.dedent('''\ + #!/usr/bin/env python + # -*- coding: utf-8 name> -*- + + """Some module docstring.""" + + + def foobar(): + pass + ''') + expected_code = textwrap.dedent('''\ + #!/usr/bin/env python + # -*- coding: utf-8 name> -*- + """Some module docstring.""" + + + def foobar(): + pass + ''') + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_code, reformatter.Reformat(llines)) + + unformatted_code = textwrap.dedent('''\ + #!/usr/bin/env python + # -*- coding: utf-8 name> -*- + """Some module docstring.""" + + + def foobar(): + pass + ''') + expected_formatted_code = textwrap.dedent('''\ + #!/usr/bin/env python + # -*- coding: utf-8 name> -*- + + """Some module docstring.""" + + + def foobar(): + pass + ''') + + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{based_on_style: pep8, ' + 'blank_line_before_module_docstring: True}')) + + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testTupleCohesion(self): + unformatted_code = textwrap.dedent("""\ + def f(): + this_is_a_very_long_function_name(an_extremely_long_variable_name, ( + 'a string that may be too long %s' % 'M15')) + """) + expected_code = textwrap.dedent("""\ + def f(): + this_is_a_very_long_function_name( + an_extremely_long_variable_name, + ('a string that may be too long %s' % 'M15')) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_code, reformatter.Reformat(llines)) + + def testSubscriptExpression(self): + code = textwrap.dedent("""\ + foo = d[not a] + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testSubscriptExpressionTerminatedByComma(self): + unformatted_code = textwrap.dedent("""\ + A[B, C,] + """) + expected_code = textwrap.dedent("""\ + A[ + B, + C, + ] + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_code, reformatter.Reformat(llines)) + + def testListWithFunctionCalls(self): + unformatted_code = textwrap.dedent("""\ + def foo(): + return [ + Bar( + xxx='some string', + yyy='another long string', + zzz='a third long string'), Bar( + xxx='some string', + yyy='another long string', + zzz='a third long string') + ] + """) + expected_code = textwrap.dedent("""\ + def foo(): + return [ + Bar(xxx='some string', + yyy='another long string', + zzz='a third long string'), + Bar(xxx='some string', + yyy='another long string', + zzz='a third long string') + ] + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_code, reformatter.Reformat(llines)) + + def testEllipses(self): + unformatted_code = textwrap.dedent("""\ + X=... + Y = X if ... else X + """) + expected_code = textwrap.dedent("""\ + X = ... + Y = X if ... else X + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_code, reformatter.Reformat(llines)) + + def testPseudoParens(self): + unformatted_code = textwrap.dedent("""\ + my_dict = { + 'key': # Some comment about the key + {'nested_key': 1, }, + } + """) + expected_code = textwrap.dedent("""\ + my_dict = { + 'key': # Some comment about the key + { + 'nested_key': 1, + }, + } + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_code, reformatter.Reformat(llines)) + + def testSplittingBeforeFirstArgumentOnFunctionCall(self): + """Tests split_before_first_argument on a function call.""" + unformatted_code = textwrap.dedent("""\ + a_very_long_function_name("long string with formatting {0:s}".format( + "mystring")) + """) + expected_formatted_code = textwrap.dedent("""\ + a_very_long_function_name( + "long string with formatting {0:s}".format("mystring")) + """) + + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{based_on_style: yapf, split_before_first_argument: True}')) + + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testSplittingBeforeFirstArgumentOnFunctionDefinition(self): + """Tests split_before_first_argument on a function definition.""" + unformatted_code = textwrap.dedent("""\ + def _GetNumberOfSecondsFromElements(year, month, day, hours, + minutes, seconds, microseconds): + return + """) + expected_formatted_code = textwrap.dedent("""\ + def _GetNumberOfSecondsFromElements( + year, month, day, hours, minutes, seconds, microseconds): + return + """) + + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{based_on_style: yapf, split_before_first_argument: True}')) + + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testSplittingBeforeFirstArgumentOnCompoundStatement(self): + """Tests split_before_first_argument on a compound statement.""" + unformatted_code = textwrap.dedent("""\ + if (long_argument_name_1 == 1 or + long_argument_name_2 == 2 or + long_argument_name_3 == 3 or + long_argument_name_4 == 4): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + if (long_argument_name_1 == 1 or long_argument_name_2 == 2 or + long_argument_name_3 == 3 or long_argument_name_4 == 4): + pass + """) + + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{based_on_style: yapf, split_before_first_argument: True}')) + + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testCoalesceBracketsOnDict(self): + """Tests coalesce_brackets on a dictionary.""" + unformatted_code = textwrap.dedent("""\ + date_time_values = ( + { + u'year': year, + u'month': month, + u'day_of_month': day_of_month, + u'hours': hours, + u'minutes': minutes, + u'seconds': seconds + } + ) + """) + expected_formatted_code = textwrap.dedent("""\ + date_time_values = ({ + u'year': year, + u'month': month, + u'day_of_month': day_of_month, + u'hours': hours, + u'minutes': minutes, + u'seconds': seconds + }) + """) + + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{based_on_style: yapf, coalesce_brackets: True}')) + + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testSplitAfterComment(self): + code = textwrap.dedent("""\ + if __name__ == "__main__": + with another_resource: + account = { + "validUntil": + int(time() + (6 * 7 * 24 * 60 * 60)) # in 6 weeks time + } + """) + + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{based_on_style: yapf, coalesce_brackets: True, ' + 'dedent_closing_brackets: true}')) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testDisableEndingCommaHeuristic(self): + code = textwrap.dedent("""\ + x = [1, 2, 3, 4, 5, 6, 7,] + """) + + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig('{based_on_style: yapf,' + ' disable_ending_comma_heuristic: True}')) + + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testDedentClosingBracketsWithTypeAnnotationExceedingLineLength(self): + unformatted_code = textwrap.dedent("""\ + def function(first_argument_xxxxxxxxxxxxxxxx=(0,), second_argument=None) -> None: + pass + + + def function(first_argument_xxxxxxxxxxxxxxxxxxxxxxx=(0,), second_argument=None) -> None: + pass + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + def function( + first_argument_xxxxxxxxxxxxxxxx=(0,), second_argument=None + ) -> None: + pass + + + def function( + first_argument_xxxxxxxxxxxxxxxxxxxxxxx=(0,), second_argument=None + ) -> None: + pass + """) # noqa + + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig('{based_on_style: yapf,' + ' dedent_closing_brackets: True}')) + + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testIndentClosingBracketsWithTypeAnnotationExceedingLineLength(self): + unformatted_code = textwrap.dedent("""\ + def function(first_argument_xxxxxxxxxxxxxxxx=(0,), second_argument=None) -> None: + pass + + + def function(first_argument_xxxxxxxxxxxxxxxxxxxxxxx=(0,), second_argument=None) -> None: + pass + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + def function( + first_argument_xxxxxxxxxxxxxxxx=(0,), second_argument=None + ) -> None: + pass + + + def function( + first_argument_xxxxxxxxxxxxxxxxxxxxxxx=(0,), second_argument=None + ) -> None: + pass + """) # noqa + + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig('{based_on_style: yapf,' + ' indent_closing_brackets: True}')) + + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testIndentClosingBracketsInFunctionCall(self): + unformatted_code = textwrap.dedent("""\ + def function(first_argument_xxxxxxxxxxxxxxxx=(0,), second_argument=None, third_and_final_argument=True): + pass + + + def function(first_argument_xxxxxxxxxxxxxxxxxxxxxxx=(0,), second_and_last_argument=None): + pass + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + def function( + first_argument_xxxxxxxxxxxxxxxx=(0,), + second_argument=None, + third_and_final_argument=True + ): + pass + + + def function( + first_argument_xxxxxxxxxxxxxxxxxxxxxxx=(0,), second_and_last_argument=None + ): + pass + """) # noqa + + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig('{based_on_style: yapf,' + ' indent_closing_brackets: True}')) + + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testIndentClosingBracketsInTuple(self): + unformatted_code = textwrap.dedent("""\ + def function(): + some_var = ('a long element', 'another long element', 'short element', 'really really long element') + return True + + def function(): + some_var = ('a couple', 'small', 'elemens') + return False + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + def function(): + some_var = ( + 'a long element', 'another long element', 'short element', + 'really really long element' + ) + return True + + + def function(): + some_var = ('a couple', 'small', 'elemens') + return False + """) # noqa + + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig('{based_on_style: yapf,' + ' indent_closing_brackets: True}')) + + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testIndentClosingBracketsInList(self): + unformatted_code = textwrap.dedent("""\ + def function(): + some_var = ['a long element', 'another long element', 'short element', 'really really long element'] + return True + + def function(): + some_var = ['a couple', 'small', 'elemens'] + return False + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + def function(): + some_var = [ + 'a long element', 'another long element', 'short element', + 'really really long element' + ] + return True + + + def function(): + some_var = ['a couple', 'small', 'elemens'] + return False + """) + + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig('{based_on_style: yapf,' + ' indent_closing_brackets: True}')) + + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testIndentClosingBracketsInDict(self): + unformatted_code = textwrap.dedent("""\ + def function(): + some_var = {1: ('a long element', 'and another really really long element that is really really amazingly long'), 2: 'another long element', 3: 'short element', 4: 'really really long element'} + return True + + def function(): + some_var = {1: 'a couple', 2: 'small', 3: 'elemens'} + return False + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + def function(): + some_var = { + 1: + ( + 'a long element', + 'and another really really long element that is really really amazingly long' + ), + 2: 'another long element', + 3: 'short element', + 4: 'really really long element' + } + return True + + + def function(): + some_var = {1: 'a couple', 2: 'small', 3: 'elemens'} + return False + """) # noqa + + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig('{based_on_style: yapf,' + ' indent_closing_brackets: True}')) + + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testMultipleDictionariesInList(self): + unformatted_code = textwrap.dedent("""\ + class A: + def b(): + d = { + "123456": [ + { + "12": "aa" + }, + { + "12": "bb" + }, + { + "12": "cc", + "1234567890": { + "1234567": [{ + "12": "dd", + "12345": "text 1" + }, { + "12": "ee", + "12345": "text 2" + }] + } + } + ] + } + """) + expected_formatted_code = textwrap.dedent("""\ + class A: + + def b(): + d = { + "123456": [{ + "12": "aa" + }, { + "12": "bb" + }, { + "12": "cc", + "1234567890": { + "1234567": [{ + "12": "dd", + "12345": "text 1" + }, { + "12": "ee", + "12345": "text 2" + }] + } + }] + } + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testForceMultilineDict_True(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig('{force_multiline_dict: true}')) + unformatted_code = textwrap.dedent("""\ + responseDict = {'childDict': {'spam': 'eggs'}} + generatedDict = {x: x for x in 'value'} + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + actual = reformatter.Reformat(llines) + expected = textwrap.dedent("""\ + responseDict = { + 'childDict': { + 'spam': 'eggs' + } + } + generatedDict = { + x: x for x in 'value' + } + """) + self.assertCodeEqual(expected, actual) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testForceMultilineDict_False(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig('{force_multiline_dict: false}')) + unformatted_code = textwrap.dedent("""\ + responseDict = {'childDict': {'spam': 'eggs'}} + generatedDict = {x: x for x in 'value'} + """) + expected_formatted_code = unformatted_code + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testWalrus(self): + unformatted_code = textwrap.dedent("""\ + if (x := len([1]*1000)>100): + print(f'{x} is pretty big' ) + """) + expected = textwrap.dedent("""\ + if (x := len([1] * 1000) > 100): + print(f'{x} is pretty big') + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected, reformatter.Reformat(llines)) + + def testStructuredPatternMatching(self): + unformatted_code = textwrap.dedent("""\ + match command.split(): + case[action ]: + ... # interpret single-verb action + case[action, obj]: + ... # interpret action, obj + """) + expected = textwrap.dedent("""\ + match command.split(): + case [action]: + ... # interpret single-verb action + case [action, obj]: + ... # interpret action, obj + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected, reformatter.Reformat(llines)) + + def testParenthesizedContextManagers(self): + unformatted_code = textwrap.dedent("""\ + with (cert_authority.cert_pem.tempfile() as ca_temp_path, patch.object(os, 'environ', os.environ | {'REQUESTS_CA_BUNDLE': ca_temp_path}),): + httpserver_url = httpserver.url_for('/resource.jar') + """) # noqa: E501 + expected = textwrap.dedent("""\ + with ( + cert_authority.cert_pem.tempfile() as ca_temp_path, + patch.object(os, 'environ', + os.environ | {'REQUESTS_CA_BUNDLE': ca_temp_path}), + ): + httpserver_url = httpserver.url_for('/resource.jar') + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected, reformatter.Reformat(llines)) + + #------tests for alignment functions-------- + def testAlignAssignBlankLineInbetween(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig('{align_assignment: true}')) + unformatted_code = textwrap.dedent("""\ + val_first = 1 + val_second += 2 + + val_third = 3 + """) + expected_formatted_code = textwrap.dedent("""\ + val_first = 1 + val_second += 2 + + val_third = 3 + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testAlignAssignCommentLineInbetween(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{align_assignment: true,' + 'align_assignment_restart_after_comments: true}')) + unformatted_code = textwrap.dedent("""\ + val_first = 1 + val_second += 2 + # comment + val_third = 3 + """) + expected_formatted_code = textwrap.dedent("""\ + val_first = 1 + val_second += 2 + # comment + val_third = 3 + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testAlignAssignContinueWithCommentLineInbetween(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{align_assignment: true,' + 'align_assignment_restart_after_comments: false}')) + unformatted_code = textwrap.dedent("""\ + val_first = 1 + val_second += 2 + # comment + val_third = 3 + """) + expected_formatted_code = textwrap.dedent("""\ + val_first = 1 + val_second += 2 + # comment + val_third = 3 + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testAlignIgnoreAssignInComment(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig('{align_assignment: true}')) + unformatted_code = textwrap.dedent("""\ + val_first = 1 + val_second += 2 + # comments should not be = aligned + val_third = 3 + """) + expected_formatted_code = textwrap.dedent("""\ + val_first = 1 + val_second += 2 + # comments should not be = aligned + val_third = 3 + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testAlignConfuseKwargs(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig('{align_assignment: true}')) + unformatted_code = textwrap.dedent("""\ + val_first = 1 + val_second += 2 + def fun(a=1): + a = 'example' + abc = '' + val_third = 3 + """) + expected_formatted_code = textwrap.dedent("""\ + val_first = 1 + val_second += 2 + + + def fun(a=1): + a = 'example' + abc = '' + + + val_third = 3 + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testAlignAssignDefLineInbetween(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig('{align_assignment: true}')) + unformatted_code = textwrap.dedent("""\ + val_first = 1 + val_second += 2 + def fun(): + a = 'example' + abc = '' + val_third = 3 + """) + expected_formatted_code = textwrap.dedent("""\ + val_first = 1 + val_second += 2 + + + def fun(): + a = 'example' + abc = '' + + + val_third = 3 + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testAlignAssignMultipleDepths(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig('{align_assignment: true}')) + unformatted_code = textwrap.dedent("""\ + if True: + val_first = 1 + val_second += 2 + val_third = 3 + val_fourth = 4 + """) + expected_formatted_code = textwrap.dedent("""\ + if True: + val_first = 1 + val_second += 2 + val_third = 3 + val_fourth = 4 + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testAlignAssignObjectWithNewLineInbetween(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig('{align_assignment: true}')) + unformatted_code = textwrap.dedent("""\ + val_first = 1 + val_second += 2 + object = { + entry1:1, + entry2:2, + entry3:3, + } + val_third = 3 + """) + expected_formatted_code = textwrap.dedent("""\ + val_first = 1 + val_second += 2 + object = { + entry1: 1, + entry2: 2, + entry3: 3, + } + val_third = 3 + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testAlignAssignWithOnlyOneAssignmentLine(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig('{align_assignment: true}')) + unformatted_code = textwrap.dedent("""\ + val_first = 1 + """) + expected_formatted_code = unformatted_code + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + +if __name__ == '__main__': + unittest.main() diff --git a/yapftests/reformatter_buganizer_test.py b/yapftests/reformatter_buganizer_test.py new file mode 100644 index 000000000..fcfd78e02 --- /dev/null +++ b/yapftests/reformatter_buganizer_test.py @@ -0,0 +1,2349 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Buganizer tests for yapf.reformatter.""" + +import textwrap +import unittest + +from yapf.yapflib import reformatter +from yapf.yapflib import style + +from yapftests import yapf_test_helper + + +class BuganizerFixes(yapf_test_helper.YAPFTest): + + @classmethod + def setUpClass(cls): + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testB137580392(self): + code = textwrap.dedent("""\ + def _create_testing_simulator_and_sink( + ) -> Tuple[_batch_simulator:_batch_simulator.BatchSimulator, + _batch_simulator.SimulationSink]: + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB73279849(self): + unformatted_code = textwrap.dedent("""\ + class A: + def _(a): + return 'hello' [ a ] + """) + expected_formatted_code = textwrap.dedent("""\ + class A: + def _(a): + return 'hello'[a] + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB122455211(self): + unformatted_code = textwrap.dedent("""\ + _zzzzzzzzzzzzzzzzzzzz = Union[sssssssssssssssssssss.pppppppppppppppp, + sssssssssssssssssssss.pppppppppppppppppppppppppppp] + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + _zzzzzzzzzzzzzzzzzzzz = Union[ + sssssssssssssssssssss.pppppppppppppppp, + sssssssssssssssssssss.pppppppppppppppppppppppppppp] + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB119300344(self): + code = textwrap.dedent("""\ + def _GenerateStatsEntries( + process_id: Text, + timestamp: Optional[rdfvalue.RDFDatetime] = None + ) -> Sequence[stats_values.StatsStoreEntry]: + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB132886019(self): + code = textwrap.dedent("""\ + X = { + 'some_dict_key': + frozenset([ + # pylint: disable=line-too-long + '//this/path/is/really/too/long/for/this/line/and/probably/should/be/split', + ]), + } + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB26521719(self): + code = textwrap.dedent("""\ + class _(): + + def _(self): + self.stubs.Set(some_type_of_arg, 'ThisIsAStringArgument', + lambda *unused_args, **unused_kwargs: fake_resolver) + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB122541552(self): + code = textwrap.dedent("""\ + # pylint: disable=g-explicit-bool-comparison,singleton-comparison + _QUERY = account.Account.query(account.Account.enabled == True) + # pylint: enable=g-explicit-bool-comparison,singleton-comparison + + + def _(): + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB124415889(self): + code = textwrap.dedent("""\ + class _(): + + def run_queue_scanners(): + return xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx( + { + components.NAME.FNOR: True, + components.NAME.DEVO: True, + }, + default=False) + + def modules_to_install(): + modules = DeepCopy(GetDef({})) + modules.update({ + 'xxxxxxxxxxxxxxxxxxxx': + GetDef('zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz', None), + }) + return modules + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB73166511(self): + code = textwrap.dedent("""\ + def _(): + if min_std is not None: + groundtruth_age_variances = tf.maximum(groundtruth_age_variances, + min_std**2) + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB118624921(self): + code = textwrap.dedent("""\ + def _(): + function_call( + alert_name='xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx', + time_delta='1h', + alert_level='bbbbbbbb', + metric='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + bork=foo) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB35417079(self): + code = textwrap.dedent("""\ + class _(): + + def _(): + X = ( + _ares_label_prefix + + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' # pylint: disable=line-too-long + 'PyTypePyTypePyTypePyTypePyTypePyTypePyTypePyTypePyTypePyTypePyTypePyTypePyType' # pytype: disable=attribute-error + 'CopybaraCopybaraCopybaraCopybaraCopybaraCopybaraCopybaraCopybaraCopybara' # copybara:strip + ) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB120047670(self): + unformatted_code = textwrap.dedent("""\ + X = { + 'NO_PING_COMPONENTS': [ + 79775, # Releases / FOO API + 79770, # Releases / BAZ API + 79780], # Releases / MUX API + + 'PING_BLOCKED_BUGS': False, + } + """) + expected_formatted_code = textwrap.dedent("""\ + X = { + 'NO_PING_COMPONENTS': [ + 79775, # Releases / FOO API + 79770, # Releases / BAZ API + 79780 + ], # Releases / MUX API + 'PING_BLOCKED_BUGS': False, + } + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB120245013(self): + unformatted_code = textwrap.dedent("""\ + class Foo(object): + def testNoAlertForShortPeriod(self, rutabaga): + self.targets[:][streamz_path,self._fillInOtherFields(streamz_path, {streamz_field_of_interest:True})] = series.Counter('1s', '+ 500x10000') + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + class Foo(object): + + def testNoAlertForShortPeriod(self, rutabaga): + self.targets[:][ + streamz_path, + self._fillInOtherFields(streamz_path, {streamz_field_of_interest: True} + )] = series.Counter('1s', '+ 500x10000') + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB117841880(self): + code = textwrap.dedent("""\ + def xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx( + aaaaaaaaaaaaaaaaaaa: AnyStr, + bbbbbbbbbbbb: Optional[Sequence[AnyStr]] = None, + cccccccccc: AnyStr = cst.DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD, + dddddddddd: Sequence[SliceDimension] = (), + eeeeeeeeeeee: AnyStr = cst.DEFAULT_CONTROL_NAME, + ffffffffffffffffffff: Optional[Callable[[pd.DataFrame], + pd.DataFrame]] = None, + gggggggggggggg: ooooooooooooo = ooooooooooooo() + ) -> pd.DataFrame: + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB111764402(self): + unformatted_code = textwrap.dedent("""\ + x = self.stubs.stub(video_classification_map, 'read_video_classifications', (lambda external_ids, **unused_kwargs: {external_id: self._get_serving_classification('video') for external_id in external_ids})) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + x = self.stubs.stub(video_classification_map, 'read_video_classifications', + (lambda external_ids, **unused_kwargs: { + external_id: self._get_serving_classification('video') + for external_id in external_ids + })) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB116825060(self): + code = textwrap.dedent("""\ + result_df = pd.DataFrame({LEARNED_CTR_COLUMN: learned_ctr}, + index=df_metrics.index) + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB112711217(self): + code = textwrap.dedent("""\ + def _(): + stats['moderated'] = ~stats.moderation_reason.isin( + approved_moderation_reasons) + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB112867548(self): + unformatted_code = textwrap.dedent("""\ + def _(): + return flask.make_response( + 'Records: {}, Problems: {}, More: {}'.format( + process_result.result_ct, process_result.problem_ct, + process_result.has_more), + httplib.ACCEPTED if process_result.has_more else httplib.OK, + {'content-type': _TEXT_CONTEXT_TYPE}) + """) + expected_formatted_code = textwrap.dedent("""\ + def _(): + return flask.make_response( + 'Records: {}, Problems: {}, More: {}'.format(process_result.result_ct, + process_result.problem_ct, + process_result.has_more), + httplib.ACCEPTED if process_result.has_more else httplib.OK, + {'content-type': _TEXT_CONTEXT_TYPE}) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB112651423(self): + unformatted_code = textwrap.dedent("""\ + def potato(feeditems, browse_use_case=None): + for item in turnip: + if kumquat: + if not feeds_variants.variants['FEEDS_LOAD_PLAYLIST_VIDEOS_FOR_ALL_ITEMS'] and item.video: + continue + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + def potato(feeditems, browse_use_case=None): + for item in turnip: + if kumquat: + if not feeds_variants.variants[ + 'FEEDS_LOAD_PLAYLIST_VIDEOS_FOR_ALL_ITEMS'] and item.video: + continue + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB80484938(self): + code = textwrap.dedent("""\ + for sssssss, aaaaaaaaaa in [ + ('ssssssssssssssssssss', 'sssssssssssssssssssssssss'), + ('nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn', + 'nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn'), + ('pppppppppppppppppppppppppppp', 'pppppppppppppppppppppppppppppppp'), + ('wwwwwwwwwwwwwwwwwwww', 'wwwwwwwwwwwwwwwwwwwwwwwww'), + ('sssssssssssssssss', 'sssssssssssssssssssssss'), + ('ggggggggggggggggggggggg', 'gggggggggggggggggggggggggggg'), + ('ggggggggggggggggg', 'gggggggggggggggggggggg'), + ('eeeeeeeeeeeeeeeeeeeeee', 'eeeeeeeeeeeeeeeeeeeeeeeeeee') + ]: + pass + + for sssssss, aaaaaaaaaa in [ + ('ssssssssssssssssssss', 'sssssssssssssssssssssssss'), + ('nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn', 'nnnnnnnnnnnnnnnnnnnnnnnnn'), + ('pppppppppppppppppppppppppppp', 'pppppppppppppppppppppppppppppppp'), + ('wwwwwwwwwwwwwwwwwwww', 'wwwwwwwwwwwwwwwwwwwwwwwww'), + ('sssssssssssssssss', 'sssssssssssssssssssssss'), + ('ggggggggggggggggggggggg', 'gggggggggggggggggggggggggggg'), + ('ggggggggggggggggg', 'gggggggggggggggggggggg'), + ('eeeeeeeeeeeeeeeeeeeeee', 'eeeeeeeeeeeeeeeeeeeeeeeeeee') + ]: + pass + + for sssssss, aaaaaaaaaa in [ + ('ssssssssssssssssssss', 'sssssssssssssssssssssssss'), + ('nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn', + 'nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn'), + ('pppppppppppppppppppppppppppp', 'pppppppppppppppppppppppppppppppp'), + ('wwwwwwwwwwwwwwwwwwww', 'wwwwwwwwwwwwwwwwwwwwwwwww'), + ('sssssssssssssssss', 'sssssssssssssssssssssss'), + ('ggggggggggggggggggggggg', 'gggggggggggggggggggggggggggg'), + ('ggggggggggggggggg', 'gggggggggggggggggggggg'), + ('eeeeeeeeeeeeeeeeeeeeee', 'eeeeeeeeeeeeeeeeeeeeeeeeeee'), + ]: + pass + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB120771563(self): + code = textwrap.dedent("""\ + class A: + + def b(): + d = { + "123456": [{ + "12": "aa" + }, { + "12": "bb" + }, { + "12": "cc", + "1234567890": { + "1234567": [{ + "12": "dd", + "12345": "text 1" + }, { + "12": "ee", + "12345": "text 2" + }] + } + }] + } + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB79462249(self): + code = textwrap.dedent("""\ + foo.bar(baz, [ + quux(thud=42), + norf, + ]) + foo.bar(baz, [ + quux(), + norf, + ]) + foo.bar(baz, quux(thud=42), aaaaaaaaaaaaaaaaaaaaaa, bbbbbbbbbbbbbbbbbbbbb, + ccccccccccccccccccc) + foo.bar( + baz, + quux(thud=42), + aaaaaaaaaaaaaaaaaaaaaa=1, + bbbbbbbbbbbbbbbbbbbbb=2, + ccccccccccccccccccc=3) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB113210278(self): + unformatted_code = textwrap.dedent("""\ + def _(): + aaaaaaaaaaa = bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.cccccccccccccccccccccccccccc(\ + eeeeeeeeeeeeeeeeeeeeeeeeee.fffffffffffffffffffffffffffffffffffffff.\ + ggggggggggggggggggggggggggggggggg.hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh()) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + def _(): + aaaaaaaaaaa = bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.cccccccccccccccccccccccccccc( + eeeeeeeeeeeeeeeeeeeeeeeeee.fffffffffffffffffffffffffffffffffffffff + .ggggggggggggggggggggggggggggggggg.hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh()) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB77923341(self): + code = textwrap.dedent("""\ + def f(): + if (aaaaaaaaaaaaaa.bbbbbbbbbbbb.ccccc <= 0 and # pytype: disable=attribute-error + ddddddddddd.eeeeeeeee == constants.FFFFFFFFFFFFFF): + raise "yo" + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB77329955(self): + code = textwrap.dedent("""\ + class _(): + + @parameterized.named_parameters( + ('ReadyExpiredSuccess', True, True, True, None, None), + ('SpannerUpdateFails', True, False, True, None, None), + ('ReadyNotExpired', False, True, True, True, None), + # ('ReadyNotExpiredNotHealthy', False, True, True, False, True), + # ('ReadyNotExpiredNotHealthyErrorFails', False, True, True, False, False + # ('ReadyNotExpiredNotHealthyUpdateFails', False, False, True, False, True + ) + def _(): + pass + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB65197969(self): + unformatted_code = textwrap.dedent("""\ + class _(): + + def _(): + return timedelta(seconds=max(float(time_scale), small_interval) * + 1.41 ** min(num_attempts, 9)) + """) + expected_formatted_code = textwrap.dedent("""\ + class _(): + + def _(): + return timedelta( + seconds=max(float(time_scale), small_interval) * + 1.41**min(num_attempts, 9)) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB65546221(self): + unformatted_code = textwrap.dedent("""\ + SUPPORTED_PLATFORMS = ( + "centos-6", + "centos-7", + "ubuntu-1204-precise", + "ubuntu-1404-trusty", + "ubuntu-1604-xenial", + "debian-7-wheezy", + "debian-8-jessie", + "debian-9-stretch",) + """) + expected_formatted_code = textwrap.dedent("""\ + SUPPORTED_PLATFORMS = ( + "centos-6", + "centos-7", + "ubuntu-1204-precise", + "ubuntu-1404-trusty", + "ubuntu-1604-xenial", + "debian-7-wheezy", + "debian-8-jessie", + "debian-9-stretch", + ) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB30500455(self): + unformatted_code = textwrap.dedent("""\ + INITIAL_SYMTAB = dict([(name, 'exception#' + name) for name in INITIAL_EXCEPTIONS + ] * [(name, 'type#' + name) for name in INITIAL_TYPES] + [ + (name, 'function#' + name) for name in INITIAL_FUNCTIONS + ] + [(name, 'const#' + name) for name in INITIAL_CONSTS]) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + INITIAL_SYMTAB = dict( + [(name, 'exception#' + name) for name in INITIAL_EXCEPTIONS] * + [(name, 'type#' + name) for name in INITIAL_TYPES] + + [(name, 'function#' + name) for name in INITIAL_FUNCTIONS] + + [(name, 'const#' + name) for name in INITIAL_CONSTS]) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB38343525(self): + code = textwrap.dedent("""\ + # This does foo. + @arg.String('some_path_to_a_file', required=True) + # This does bar. + @arg.String('some_path_to_a_file', required=True) + def f(): + print(1) + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB37099651(self): + unformatted_code = textwrap.dedent("""\ + _MEMCACHE = lazy.MakeLazy( + # pylint: disable=g-long-lambda + lambda: function.call.mem.clients(FLAGS.some_flag_thingy, default_namespace=_LAZY_MEM_NAMESPACE, allow_pickle=True) + # pylint: enable=g-long-lambda + ) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + _MEMCACHE = lazy.MakeLazy( + # pylint: disable=g-long-lambda + lambda: function.call.mem.clients( + FLAGS.some_flag_thingy, + default_namespace=_LAZY_MEM_NAMESPACE, + allow_pickle=True) + # pylint: enable=g-long-lambda + ) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB33228502(self): + unformatted_code = textwrap.dedent("""\ + def _(): + success_rate_stream_table = module.Precompute( + query_function=module.DefineQueryFunction( + name='Response error ratio', + expression=((m.Fetch( + m.Raw('monarch.BorgTask', + '/corp/travel/trips2/dispatcher/email/response'), + {'borg_job': module_config.job, 'metric:response_type': 'SUCCESS'}), + m.Fetch(m.Raw('monarch.BorgTask', '/corp/travel/trips2/dispatcher/email/response'), {'borg_job': module_config.job})) + | m.Window(m.Delta('1h')) + | m.Join('successes', 'total') + | m.Point(m.VAL['successes'] / m.VAL['total'])))) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + def _(): + success_rate_stream_table = module.Precompute( + query_function=module.DefineQueryFunction( + name='Response error ratio', + expression=( + (m.Fetch( + m.Raw('monarch.BorgTask', + '/corp/travel/trips2/dispatcher/email/response'), { + 'borg_job': module_config.job, + 'metric:response_type': 'SUCCESS' + }), + m.Fetch( + m.Raw('monarch.BorgTask', + '/corp/travel/trips2/dispatcher/email/response'), + {'borg_job': module_config.job})) + | m.Window(m.Delta('1h')) + | m.Join('successes', 'total') + | m.Point(m.VAL['successes'] / m.VAL['total'])))) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB30394228(self): + code = textwrap.dedent("""\ + class _(): + + def _(self): + return some.randome.function.calling( + wf, None, alert.Format(alert.subject, alert=alert, threshold=threshold), + alert.Format(alert.body, alert=alert, threshold=threshold), + alert.html_formatting) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB65246454(self): + unformatted_code = textwrap.dedent("""\ + class _(): + + def _(self): + self.assertEqual({i.id + for i in successful_instances}, + {i.id + for i in self._statuses.successful_instances}) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + class _(): + + def _(self): + self.assertEqual({i.id for i in successful_instances}, + {i.id for i in self._statuses.successful_instances}) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB67935450(self): + unformatted_code = textwrap.dedent("""\ + def _(): + return ( + (Gauge( + metric='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + group_by=group_by + ['metric:process_name'], + metric_filter={'metric:process_name': process_name_re}), + Gauge( + metric='bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + group_by=group_by + ['metric:process_name'], + metric_filter={'metric:process_name': process_name_re})) + | expr.Join( + left_name='start', left_default=0, right_name='end', right_default=0) + | m.Point( + m.Cond(m.VAL['end'] != 0, m.VAL['end'], k.TimestampMicros() / + 1000000L) - m.Cond(m.VAL['start'] != 0, m.VAL['start'], + m.TimestampMicros() / 1000000L))) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + def _(): + return ( + (Gauge( + metric='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + group_by=group_by + ['metric:process_name'], + metric_filter={'metric:process_name': process_name_re}), + Gauge( + metric='bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + group_by=group_by + ['metric:process_name'], + metric_filter={'metric:process_name': process_name_re})) + | expr.Join( + left_name='start', left_default=0, right_name='end', right_default=0) + | m.Point( + m.Cond(m.VAL['end'] != 0, m.VAL['end'], + k.TimestampMicros() / 1000000L) - + m.Cond(m.VAL['start'] != 0, m.VAL['start'], + m.TimestampMicros() / 1000000L))) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB66011084(self): + unformatted_code = textwrap.dedent("""\ + X = { + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa": # Comment 1. + ([] if True else [ # Comment 2. + "bbbbbbbbbbbbbbbbbbb", # Comment 3. + "cccccccccccccccccccccccc", # Comment 4. + "ddddddddddddddddddddddddd", # Comment 5. + "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", # Comment 6. + "fffffffffffffffffffffffffffffff", # Comment 7. + "ggggggggggggggggggggggggggg", # Comment 8. + "hhhhhhhhhhhhhhhhhh", # Comment 9. + ]), + } + """) + expected_formatted_code = textwrap.dedent("""\ + X = { + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa": # Comment 1. + ([] if True else [ # Comment 2. + "bbbbbbbbbbbbbbbbbbb", # Comment 3. + "cccccccccccccccccccccccc", # Comment 4. + "ddddddddddddddddddddddddd", # Comment 5. + "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", # Comment 6. + "fffffffffffffffffffffffffffffff", # Comment 7. + "ggggggggggggggggggggggggggg", # Comment 8. + "hhhhhhhhhhhhhhhhhh", # Comment 9. + ]), + } + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB67455376(self): + unformatted_code = textwrap.dedent("""\ + sponge_ids.extend(invocation.id() for invocation in self._client.GetInvocationsByLabels(labels)) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + sponge_ids.extend(invocation.id() + for invocation in self._client.GetInvocationsByLabels(labels)) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB35210351(self): + unformatted_code = textwrap.dedent("""\ + def _(): + config.AnotherRuleThing( + 'the_title_to_the_thing_here', + {'monitorname': 'firefly', + 'service': ACCOUNTING_THING, + 'severity': 'the_bug', + 'monarch_module_name': alerts.TheLabel(qa_module_regexp, invert=True)}, + fanout, + alerts.AlertUsToSomething( + GetTheAlertToIt('the_title_to_the_thing_here'), + GetNotificationTemplate('your_email_here'))) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + def _(): + config.AnotherRuleThing( + 'the_title_to_the_thing_here', { + 'monitorname': 'firefly', + 'service': ACCOUNTING_THING, + 'severity': 'the_bug', + 'monarch_module_name': alerts.TheLabel(qa_module_regexp, invert=True) + }, fanout, + alerts.AlertUsToSomething( + GetTheAlertToIt('the_title_to_the_thing_here'), + GetNotificationTemplate('your_email_here'))) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB34774905(self): + unformatted_code = textwrap.dedent("""\ + x=[VarExprType(ir_name=IrName( value='x', + expr_type=UnresolvedAttrExprType( atom=UnknownExprType(), attr_name=IrName( + value='x', expr_type=UnknownExprType(), usage='UNKNOWN', fqn=None, + astn=None), usage='REF'), usage='ATTR', fqn='.x', astn=None))] + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + x = [ + VarExprType( + ir_name=IrName( + value='x', + expr_type=UnresolvedAttrExprType( + atom=UnknownExprType(), + attr_name=IrName( + value='x', + expr_type=UnknownExprType(), + usage='UNKNOWN', + fqn=None, + astn=None), + usage='REF'), + usage='ATTR', + fqn='.x', + astn=None)) + ] + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB65176185(self): + code = textwrap.dedent("""\ + xx = zip(*[(a, b) for (a, b, c) in yy]) + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB35210166(self): + unformatted_code = textwrap.dedent("""\ + def _(): + query = ( + m.Fetch(n.Raw('monarch.BorgTask', '/proc/container/memory/usage'), { 'borg_user': borguser, 'borg_job': jobname }) + | o.Window(m.Align('5m')) | p.GroupBy(['borg_user', 'borg_job', 'borg_cell'], q.Mean())) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + def _(): + query = ( + m.Fetch( + n.Raw('monarch.BorgTask', '/proc/container/memory/usage'), { + 'borg_user': borguser, + 'borg_job': jobname + }) + | o.Window(m.Align('5m')) + | p.GroupBy(['borg_user', 'borg_job', 'borg_cell'], q.Mean())) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB32167774(self): + unformatted_code = textwrap.dedent("""\ + X = ( + 'is_official', + 'is_cover', + 'is_remix', + 'is_instrumental', + 'is_live', + 'has_lyrics', + 'is_album', + 'is_compilation',) + """) + expected_formatted_code = textwrap.dedent("""\ + X = ( + 'is_official', + 'is_cover', + 'is_remix', + 'is_instrumental', + 'is_live', + 'has_lyrics', + 'is_album', + 'is_compilation', + ) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB66912275(self): + unformatted_code = textwrap.dedent("""\ + def _(): + with self.assertRaisesRegexp(errors.HttpError, 'Invalid'): + patch_op = api_client.forwardingRules().patch( + project=project_id, + region=region, + forwardingRule=rule_name, + body={'fingerprint': base64.urlsafe_b64encode('invalid_fingerprint')}).execute() + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + def _(): + with self.assertRaisesRegexp(errors.HttpError, 'Invalid'): + patch_op = api_client.forwardingRules().patch( + project=project_id, + region=region, + forwardingRule=rule_name, + body={ + 'fingerprint': base64.urlsafe_b64encode('invalid_fingerprint') + }).execute() + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB67312284(self): + code = textwrap.dedent("""\ + def _(): + self.assertEqual( + [u'to be published 2', u'to be published 1', u'to be published 0'], + [el.text for el in page.first_column_tds]) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB65241516(self): + unformatted_code = textwrap.dedent("""\ + checkpoint_files = gfile.Glob(os.path.join(TrainTraceDir(unit_key, "*", "*"), embedding_model.CHECKPOINT_FILENAME + "-*")) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + checkpoint_files = gfile.Glob( + os.path.join( + TrainTraceDir(unit_key, "*", "*"), + embedding_model.CHECKPOINT_FILENAME + "-*")) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB37460004(self): + code = textwrap.dedent("""\ + assert all(s not in (_SENTINEL, None) for s in nested_schemas + ), 'Nested schemas should never contain None/_SENTINEL' + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB36806207(self): + code = textwrap.dedent("""\ + def _(): + linearity_data = [[row] for row in [ + "%.1f mm" % (np.mean(linearity_values["pos_error"]) * 1000.0), + "%.1f mm" % (np.max(linearity_values["pos_error"]) * 1000.0), + "%.1f mm" % (np.mean(linearity_values["pos_error_chunk_mean"]) * 1000.0), + "%.1f mm" % (np.max(linearity_values["pos_error_chunk_max"]) * 1000.0), + "%.1f deg" % math.degrees(np.mean(linearity_values["rot_noise"])), + "%.1f deg" % math.degrees(np.max(linearity_values["rot_noise"])), + "%.1f deg" % math.degrees(np.mean(linearity_values["rot_drift"])), + "%.1f deg" % math.degrees(np.max(linearity_values["rot_drift"])), + "%.1f%%" % (np.max(linearity_values["pos_discontinuity"]) * 100.0), + "%.1f%%" % (np.max(linearity_values["rot_discontinuity"]) * 100.0) + ]] + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB36215507(self): + code = textwrap.dedent("""\ + class X(): + + def _(): + aaaaaaaaaaaaa._bbbbbbbbbbbbbbbbbbbbbbbbbbbbbb( + mmmmmmmmmmmmm, nnnnn, ooooooooo, + _(ppppppppppppppppppppppppppppppppppppp), + *(qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq), + **(qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq)) + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB35212469(self): + unformatted_code = textwrap.dedent("""\ + def _(): + X = { + 'retain': { + 'loadtest': # This is a comment in the middle of a dictionary entry + ('/some/path/to/a/file/that/is/needed/by/this/process') + } + } + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + def _(): + X = { + 'retain': { + 'loadtest': # This is a comment in the middle of a dictionary entry + ('/some/path/to/a/file/that/is/needed/by/this/process') + } + } + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB31063453(self): + unformatted_code = textwrap.dedent("""\ + def _(): + while ((not mpede_proc) or ((time_time() - last_modified) < FLAGS_boot_idle_timeout)): + pass + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + def _(): + while ((not mpede_proc) or + ((time_time() - last_modified) < FLAGS_boot_idle_timeout)): + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB35021894(self): + unformatted_code = textwrap.dedent("""\ + def _(): + labelacl = Env(qa={ + 'read': 'name/some-type-of-very-long-name-for-reading-perms', + 'modify': 'name/some-other-type-of-very-long-name-for-modifying' + }, + prod={ + 'read': 'name/some-type-of-very-long-name-for-reading-perms', + 'modify': 'name/some-other-type-of-very-long-name-for-modifying' + }) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + def _(): + labelacl = Env( + qa={ + 'read': 'name/some-type-of-very-long-name-for-reading-perms', + 'modify': 'name/some-other-type-of-very-long-name-for-modifying' + }, + prod={ + 'read': 'name/some-type-of-very-long-name-for-reading-perms', + 'modify': 'name/some-other-type-of-very-long-name-for-modifying' + }) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB34682902(self): + unformatted_code = textwrap.dedent("""\ + logging.info("Mean angular velocity norm: %.3f", np.linalg.norm(np.mean(ang_vel_arr, axis=0))) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + logging.info("Mean angular velocity norm: %.3f", + np.linalg.norm(np.mean(ang_vel_arr, axis=0))) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB33842726(self): + unformatted_code = textwrap.dedent("""\ + class _(): + def _(): + hints.append(('hg tag -f -l -r %s %s # %s' % (short(ctx.node( + )), candidatetag, firstline))[:78]) + """) + expected_formatted_code = textwrap.dedent("""\ + class _(): + def _(): + hints.append(('hg tag -f -l -r %s %s # %s' % + (short(ctx.node()), candidatetag, firstline))[:78]) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB32931780(self): + unformatted_code = textwrap.dedent("""\ + environments = { + 'prod': { + # this is a comment before the first entry. + 'entry one': + 'an entry.', + # this is the comment before the second entry. + 'entry number 2.': + 'something', + # this is the comment before the third entry and it's a doozy. So big! + 'who': + 'allin', + # This is an entry that has a dictionary in it. It's ugly + 'something': { + 'page': ['this-is-a-page@xxxxxxxx.com', 'something-for-eml@xxxxxx.com'], + 'bug': ['bugs-go-here5300@xxxxxx.com'], + 'email': ['sometypeof-email@xxxxxx.com'], + }, + # a short comment + 'yolo!!!!!': + 'another-email-address@xxxxxx.com', + # this entry has an implicit string concatenation + 'implicit': + 'https://this-is-very-long.url-addr.com/' + '?something=something%20some%20more%20stuff..', + # A more normal entry. + '.....': + 'this is an entry', + } + } + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + environments = { + 'prod': { + # this is a comment before the first entry. + 'entry one': 'an entry.', + # this is the comment before the second entry. + 'entry number 2.': 'something', + # this is the comment before the third entry and it's a doozy. So big! + 'who': 'allin', + # This is an entry that has a dictionary in it. It's ugly + 'something': { + 'page': [ + 'this-is-a-page@xxxxxxxx.com', 'something-for-eml@xxxxxx.com' + ], + 'bug': ['bugs-go-here5300@xxxxxx.com'], + 'email': ['sometypeof-email@xxxxxx.com'], + }, + # a short comment + 'yolo!!!!!': 'another-email-address@xxxxxx.com', + # this entry has an implicit string concatenation + 'implicit': 'https://this-is-very-long.url-addr.com/' + '?something=something%20some%20more%20stuff..', + # A more normal entry. + '.....': 'this is an entry', + } + } + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB33047408(self): + code = textwrap.dedent("""\ + def _(): + for sort in (sorts or []): + request['sorts'].append({ + 'field': { + 'user_field': sort + }, + 'order': 'ASCENDING' + }) + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB32714745(self): + code = textwrap.dedent("""\ + class _(): + + def _BlankDefinition(): + '''Return a generic blank dictionary for a new field.''' + return { + 'type': '', + 'validation': '', + 'name': 'fieldname', + 'label': 'Field Label', + 'help': '', + 'initial': '', + 'required': False, + 'required_msg': 'Required', + 'invalid_msg': 'Please enter a valid value', + 'options': { + 'regex': '', + 'widget_attr': '', + 'choices_checked': '', + 'choices_count': '', + 'choices': {} + }, + 'isnew': True, + 'dirty': False, + } + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB32737279(self): + unformatted_code = textwrap.dedent("""\ + here_is_a_dict = { + 'key': + # Comment. + 'value' + } + """) + expected_formatted_code = textwrap.dedent("""\ + here_is_a_dict = { + 'key': # Comment. + 'value' + } + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB32570937(self): + code = textwrap.dedent("""\ + def _(): + if (job_message.ball not in ('*', ball) or + job_message.call not in ('*', call) or + job_message.mall not in ('*', job_name)): + return False + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB31937033(self): + code = textwrap.dedent("""\ + class _(): + + def __init__(self, metric, fields_cb=None): + self._fields_cb = fields_cb or (lambda *unused_args, **unused_kwargs: {}) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB31911533(self): + code = textwrap.dedent("""\ + class _(): + + @parameterized.NamedParameters( + ('IncludingModInfoWithHeaderList', AAAA, aaaa), + ('IncludingModInfoWithoutHeaderList', BBBB, bbbbb), + ('ExcludingModInfoWithHeaderList', CCCCC, cccc), + ('ExcludingModInfoWithoutHeaderList', DDDDD, ddddd), + ) + def _(): + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB31847238(self): + unformatted_code = textwrap.dedent("""\ + class _(): + + def aaaaa(self, bbbbb, cccccccccccccc=None): # TODO(who): pylint: disable=unused-argument + return 1 + + def xxxxx(self, yyyyy, zzzzzzzzzzzzzz=None): # A normal comment that runs over the column limit. + return 1 + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + class _(): + + def aaaaa(self, bbbbb, cccccccccccccc=None): # TODO(who): pylint: disable=unused-argument + return 1 + + def xxxxx( + self, + yyyyy, + zzzzzzzzzzzzzz=None): # A normal comment that runs over the column limit. + return 1 + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB30760569(self): + unformatted_code = textwrap.dedent("""\ + {'1234567890123456789012345678901234567890123456789012345678901234567890': + '1234567890123456789012345678901234567890'} + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + { + '1234567890123456789012345678901234567890123456789012345678901234567890': + '1234567890123456789012345678901234567890' + } + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB26034238(self): + unformatted_code = textwrap.dedent("""\ + class Thing: + + def Function(self): + thing.Scrape('/aaaaaaaaa/bbbbbbbbbb/ccccc/dddd/eeeeeeeeeeeeee/ffffffffffffff').AndReturn(42) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + class Thing: + + def Function(self): + thing.Scrape( + '/aaaaaaaaa/bbbbbbbbbb/ccccc/dddd/eeeeeeeeeeeeee/ffffffffffffff' + ).AndReturn(42) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB30536435(self): + unformatted_code = textwrap.dedent("""\ + def main(unused_argv): + if True: + if True: + aaaaaaaaaaa.comment('import-from[{}] {} {}'.format( + bbbbbbbbb.usage, + ccccccccc.within, + imports.ddddddddddddddddddd(name_item.ffffffffffffffff))) + """) + expected_formatted_code = textwrap.dedent("""\ + def main(unused_argv): + if True: + if True: + aaaaaaaaaaa.comment('import-from[{}] {} {}'.format( + bbbbbbbbb.usage, ccccccccc.within, + imports.ddddddddddddddddddd(name_item.ffffffffffffffff))) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB30442148(self): + unformatted_code = textwrap.dedent("""\ + def lulz(): + return (some_long_module_name.SomeLongClassName. + some_long_attribute_name.some_long_method_name()) + """) + expected_formatted_code = textwrap.dedent("""\ + def lulz(): + return (some_long_module_name.SomeLongClassName.some_long_attribute_name + .some_long_method_name()) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB26868213(self): + unformatted_code = textwrap.dedent("""\ + def _(): + xxxxxxxxxxxxxxxxxxx = { + 'ssssss': {'ddddd': 'qqqqq', + 'p90': aaaaaaaaaaaaaaaaa, + 'p99': bbbbbbbbbbbbbbbbb, + 'lllllllllllll': yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy(),}, + 'bbbbbbbbbbbbbbbbbbbbbbbbbbbb': { + 'ddddd': 'bork bork bork bo', + 'p90': wwwwwwwwwwwwwwwww, + 'p99': wwwwwwwwwwwwwwwww, + 'lllllllllllll': None, # use the default + } + } + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + def _(): + xxxxxxxxxxxxxxxxxxx = { + 'ssssss': { + 'ddddd': 'qqqqq', + 'p90': aaaaaaaaaaaaaaaaa, + 'p99': bbbbbbbbbbbbbbbbb, + 'lllllllllllll': yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy(), + }, + 'bbbbbbbbbbbbbbbbbbbbbbbbbbbb': { + 'ddddd': 'bork bork bork bo', + 'p90': wwwwwwwwwwwwwwwww, + 'p99': wwwwwwwwwwwwwwwww, + 'lllllllllllll': None, # use the default + } + } + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB30173198(self): + code = textwrap.dedent("""\ + class _(): + + def _(): + self.assertFalse( + evaluation_runner.get_larps_in_eval_set('these_arent_the_larps')) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB29908765(self): + code = textwrap.dedent("""\ + class _(): + + def __repr__(self): + return '' % ( + self._id, self._stub._stub.rpc_channel().target()) # pylint:disable=protected-access + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB30087362(self): + code = textwrap.dedent("""\ + def _(): + for s in sorted(env['foo']): + bar() + # This is a comment + + # This is another comment + foo() + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB30087363(self): + code = textwrap.dedent("""\ + if False: + bar() + # This is a comment + # This is another comment + elif True: + foo() + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB29093579(self): + unformatted_code = textwrap.dedent("""\ + def _(): + _xxxxxxxxxxxxxxx(aaaaaaaa, bbbbbbbbbbbbbb.cccccccccc[ + dddddddddddddddddddddddddddd.eeeeeeeeeeeeeeeeeeeeee.fffffffffffffffffffff]) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + def _(): + _xxxxxxxxxxxxxxx( + aaaaaaaa, + bbbbbbbbbbbbbb.cccccccccc[dddddddddddddddddddddddddddd + .eeeeeeeeeeeeeeeeeeeeee.fffffffffffffffffffff]) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB26382315(self): + code = textwrap.dedent("""\ + @hello_world + # This is a first comment + + # Comment + def foo(): + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB27616132(self): + unformatted_code = textwrap.dedent("""\ + if True: + query.fetch_page.assert_has_calls([ + mock.call(100, + start_cursor=None), + mock.call(100, + start_cursor=cursor_1), + mock.call(100, + start_cursor=cursor_2), + ]) + """) + expected_formatted_code = textwrap.dedent("""\ + if True: + query.fetch_page.assert_has_calls([ + mock.call(100, start_cursor=None), + mock.call(100, start_cursor=cursor_1), + mock.call(100, start_cursor=cursor_2), + ]) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB27590179(self): + unformatted_code = textwrap.dedent("""\ + if True: + if True: + self.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa = ( + { True: + self.bbb.cccccccccc(ddddddddddddddddddddddd.eeeeeeeeeeeeeeeeeeeeee), + False: + self.bbb.cccccccccc(ddddddddddddddddddddddd.eeeeeeeeeeeeeeeeeeeeee) + }) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + if True: + if True: + self.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa = ({ + True: + self.bbb.cccccccccc(ddddddddddddddddddddddd.eeeeeeeeeeeeeeeeeeeeee), + False: + self.bbb.cccccccccc(ddddddddddddddddddddddd.eeeeeeeeeeeeeeeeeeeeee) + }) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB27266946(self): + unformatted_code = textwrap.dedent("""\ + def _(): + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa = (self.bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.cccccccccccccccccccccccccccccccccccc) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + def _(): + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa = ( + self.bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb + .cccccccccccccccccccccccccccccccccccc) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB25505359(self): + code = textwrap.dedent("""\ + _EXAMPLE = { + 'aaaaaaaaaaaaaa': [{ + 'bbbb': 'cccccccccccccccccccccc', + 'dddddddddddd': [] + }, { + 'bbbb': 'ccccccccccccccccccc', + 'dddddddddddd': [] + }] + } + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB25324261(self): + code = textwrap.dedent("""\ + aaaaaaaaa = set(bbbb.cccc + for ddd in eeeeee.fffffffffff.gggggggggggggggg + for cccc in ddd.specification) + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB25136704(self): + code = textwrap.dedent("""\ + class f: + + def test(self): + self.bbbbbbb[0]['aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', { + 'xxxxxx': 'yyyyyy' + }] = cccccc.ddd('1m', '10x1+1') + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB25165602(self): + code = textwrap.dedent("""\ + def f(): + ids = {u: i for u, i in zip(self.aaaaa, xrange(42, 42 + len(self.aaaaaa)))} + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB25157123(self): + code = textwrap.dedent("""\ + def ListArgs(): + FairlyLongMethodName([relatively_long_identifier_for_a_list], + another_argument_with_a_long_identifier) + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB25136820(self): + unformatted_code = textwrap.dedent("""\ + def foo(): + return collections.OrderedDict({ + # Preceding comment. + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa': + '$bbbbbbbbbbbbbbbbbbbbbbbb', + }) + """) + expected_formatted_code = textwrap.dedent("""\ + def foo(): + return collections.OrderedDict({ + # Preceding comment. + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa': + '$bbbbbbbbbbbbbbbbbbbbbbbb', + }) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB25131481(self): + unformatted_code = textwrap.dedent("""\ + APPARENT_ACTIONS = ('command_type', { + 'materialize': lambda x: some_type_of_function('materialize ' + x.command_def), + '#': lambda x: x # do nothing + }) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + APPARENT_ACTIONS = ( + 'command_type', + { + 'materialize': + lambda x: some_type_of_function('materialize ' + x.command_def), + '#': + lambda x: x # do nothing + }) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB23445244(self): + unformatted_code = textwrap.dedent("""\ + def foo(): + if True: + return xxxxxxxxxxxxxxxx( + command, + extra_env={ + "OOOOOOOOOOOOOOOOOOOOO": FLAGS.zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz, + "PPPPPPPPPPPPPPPPPPPPP": + FLAGS.aaaaaaaaaaaaaa + FLAGS.bbbbbbbbbbbbbbbbbbb, + }) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + def foo(): + if True: + return xxxxxxxxxxxxxxxx( + command, + extra_env={ + "OOOOOOOOOOOOOOOOOOOOO": + FLAGS.zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz, + "PPPPPPPPPPPPPPPPPPPPP": + FLAGS.aaaaaaaaaaaaaa + FLAGS.bbbbbbbbbbbbbbbbbbb, + }) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB20559654(self): + unformatted_code = textwrap.dedent("""\ + class A(object): + + def foo(self): + unused_error, result = server.Query( + ['AA BBBB CCC DDD EEEEEEEE X YY ZZZZ FFF EEE AAAAAAAA'], + aaaaaaaaaaa=True, bbbbbbbb=None) + """) + expected_formatted_code = textwrap.dedent("""\ + class A(object): + + def foo(self): + unused_error, result = server.Query( + ['AA BBBB CCC DDD EEEEEEEE X YY ZZZZ FFF EEE AAAAAAAA'], + aaaaaaaaaaa=True, + bbbbbbbb=None) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB23943842(self): + unformatted_code = textwrap.dedent("""\ + class F(): + def f(): + self.assertDictEqual( + accounts, { + 'foo': + {'account': 'foo', + 'lines': 'l1\\nl2\\nl3\\n1 line(s) were elided.'}, + 'bar': {'account': 'bar', + 'lines': 'l5\\nl6\\nl7'}, + 'wiz': {'account': 'wiz', + 'lines': 'l8'} + }) + """) + expected_formatted_code = textwrap.dedent("""\ + class F(): + + def f(): + self.assertDictEqual( + accounts, { + 'foo': { + 'account': 'foo', + 'lines': 'l1\\nl2\\nl3\\n1 line(s) were elided.' + }, + 'bar': { + 'account': 'bar', + 'lines': 'l5\\nl6\\nl7' + }, + 'wiz': { + 'account': 'wiz', + 'lines': 'l8' + } + }) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB20551180(self): + unformatted_code = textwrap.dedent("""\ + def foo(): + if True: + return (struct.pack('aaaa', bbbbbbbbbb, ccccccccccccccc, dddddddd) + eeeeeee) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + def foo(): + if True: + return (struct.pack('aaaa', bbbbbbbbbb, ccccccccccccccc, dddddddd) + + eeeeeee) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB23944849(self): + unformatted_code = textwrap.dedent("""\ + class A(object): + def xxxxxxxxx(self, aaaaaaa, bbbbbbb=ccccccccccc, dddddd=300, eeeeeeeeeeeeee=None, fffffffffffffff=0): + pass + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + class A(object): + + def xxxxxxxxx(self, + aaaaaaa, + bbbbbbb=ccccccccccc, + dddddd=300, + eeeeeeeeeeeeee=None, + fffffffffffffff=0): + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB23935890(self): + unformatted_code = textwrap.dedent("""\ + class F(): + def functioni(self, aaaaaaa, bbbbbbb, cccccc, dddddddddddddd, eeeeeeeeeeeeeee): + pass + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + class F(): + + def functioni(self, aaaaaaa, bbbbbbb, cccccc, dddddddddddddd, + eeeeeeeeeeeeeee): + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB28414371(self): + code = textwrap.dedent("""\ + def _(): + return ((m.fffff( + m.rrr('mmmmmmmmmmmmmmmm', 'ssssssssssssssssssssssssss'), ffffffffffffffff) + | m.wwwwww(m.ddddd('1h')) + | m.ggggggg(bbbbbbbbbbbbbbb) + | m.ppppp( + (1 - m.ffffffffffffffff(llllllllllllllllllllll * 1000000, m.vvv)) + * m.ddddddddddddddddd(m.vvv)), + m.fffff( + m.rrr('mmmmmmmmmmmmmmmm', 'sssssssssssssssssssssss'), + dict( + ffffffffffffffff, **{ + 'mmmmmm:ssssss': + m.rrrrrrrrrrr('|'.join(iiiiiiiiiiiiii), iiiiii=True) + })) + | m.wwwwww(m.rrrr('1h')) + | m.ggggggg(bbbbbbbbbbbbbbb)) + | m.jjjj() + | m.ppppp(m.vvv[0] + m.vvv[1])) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB20127686(self): + code = textwrap.dedent("""\ + def f(): + if True: + return ((m.fffff( + m.rrr('xxxxxxxxxxxxxxxx', + 'yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy'), + mmmmmmmm) + | m.wwwwww(m.rrrr(self.tttttttttt, self.mmmmmmmmmmmmmmmmmmmmm)) + | m.ggggggg(self.gggggggg, m.sss()), m.fffff('aaaaaaaaaaaaaaaa') + | m.wwwwww(m.ddddd(self.tttttttttt, self.mmmmmmmmmmmmmmmmmmmmm)) + | m.ggggggg(self.gggggggg)) + | m.jjjj() + | m.ppppp(m.VAL[0] / m.VAL[1])) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB20016122(self): + unformatted_code = textwrap.dedent("""\ + from a_very_long_or_indented_module_name_yada_yada import (long_argument_1, + long_argument_2) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + from a_very_long_or_indented_module_name_yada_yada import ( + long_argument_1, long_argument_2) + """) + + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{based_on_style: pep8, split_penalty_import_names: 350}')) + + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreatePEP8Style()) + + code = textwrap.dedent("""\ + class foo(): + + def __eq__(self, other): + return (isinstance(other, type(self)) + and self.xxxxxxxxxxx == other.xxxxxxxxxxx + and self.xxxxxxxx == other.xxxxxxxx + and self.aaaaaaaaaaaa == other.aaaaaaaaaaaa + and self.bbbbbbbbbbb == other.bbbbbbbbbbb + and self.ccccccccccccccccc == other.ccccccccccccccccc + and self.ddddddddddddddddddddddd == other.ddddddddddddddddddddddd + and self.eeeeeeeeeeee == other.eeeeeeeeeeee + and self.ffffffffffffff == other.time_completed + and self.gggggg == other.gggggg and self.hhh == other.hhh + and len(self.iiiiiiii) == len(other.iiiiiiii) + and all(jjjjjjj in other.iiiiiiii for jjjjjjj in self.iiiiiiii)) + """) # noqa + + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig('{based_on_style: yapf, ' + 'split_before_logical_operator: True}')) + + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testB22527411(self): + unformatted_code = textwrap.dedent("""\ + def f(): + if True: + aaaaaa.bbbbbbbbbbbbbbbbbbbb[-1].cccccccccccccc.ddd().eeeeeeee(ffffffffffffff) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + def f(): + if True: + aaaaaa.bbbbbbbbbbbbbbbbbbbb[-1].cccccccccccccc.ddd().eeeeeeee( + ffffffffffffff) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB20849933(self): + unformatted_code = textwrap.dedent("""\ + def main(unused_argv): + if True: + aaaaaaaa = { + 'xxx': '%s/cccccc/ddddddddddddddddddd.jar' % + (eeeeee.FFFFFFFFFFFFFFFFFF), + } + """) + expected_formatted_code = textwrap.dedent("""\ + def main(unused_argv): + if True: + aaaaaaaa = { + 'xxx': + '%s/cccccc/ddddddddddddddddddd.jar' % (eeeeee.FFFFFFFFFFFFFFFFFF), + } + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB20813997(self): + code = textwrap.dedent("""\ + def myfunc_1(): + myarray = numpy.zeros((2, 2, 2)) + print(myarray[:, 1, :]) + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB20605036(self): + code = textwrap.dedent("""\ + foo = { + 'aaaa': { + # A comment for no particular reason. + 'xxxxxxxx': 'bbbbbbbbb', + 'yyyyyyyyyyyyyyyyyy': 'cccccccccccccccccccccccccccccc' + 'dddddddddddddddddddddddddddddddddddddddddd', + } + } + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB20562732(self): + code = textwrap.dedent("""\ + foo = [ + # Comment about first list item + 'First item', + # Comment about second list item + 'Second item', + ] + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB20128830(self): + code = textwrap.dedent("""\ + a = { + 'xxxxxxxxxxxxxxxxxxxx': { + 'aaaa': + 'mmmmmmm', + 'bbbbb': + 'mmmmmmmmmmmmmmmmmmmmm', + 'cccccccccc': [ + 'nnnnnnnnnnn', + 'ooooooooooo', + 'ppppppppppp', + 'qqqqqqqqqqq', + ], + }, + } + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB20073838(self): + code = textwrap.dedent("""\ + class DummyModel(object): + + def do_nothing(self, class_1_count): + if True: + class_0_count = num_votes - class_1_count + return ('{class_0_name}={class_0_count}, {class_1_name}={class_1_count}' + .format( + class_0_name=self.class_0_name, + class_0_count=class_0_count, + class_1_name=self.class_1_name, + class_1_count=class_1_count)) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB19626808(self): + code = textwrap.dedent("""\ + if True: + aaaaaaaaaaaaaaaaaaaaaaa.bbbbbbbbb( + 'ccccccccccc', ddddddddd='eeeee').fffffffff([ggggggggggggggggggggg]) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB19547210(self): + code = textwrap.dedent("""\ + while True: + if True: + if True: + if True: + if xxxxxxxxxxxx.yyyyyyy(aa).zzzzzzz() not in ( + xxxxxxxxxxxx.yyyyyyyyyyyyyy.zzzzzzzz, + xxxxxxxxxxxx.yyyyyyyyyyyyyy.zzzzzzzz): + continue + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB19377034(self): + code = textwrap.dedent("""\ + def f(): + if (aaaaaaaaaaaaaaa.start >= aaaaaaaaaaaaaaa.end or + bbbbbbbbbbbbbbb.start >= bbbbbbbbbbbbbbb.end): + return False + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB19372573(self): + code = textwrap.dedent("""\ + def f(): + if a: return 42 + while True: + if b: continue + if c: break + return 0 + """) + + try: + style.SetGlobalStyle(style.CreatePEP8Style()) + + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testB19353268(self): + code = textwrap.dedent("""\ + a = {1, 2, 3}[x] + b = {'foo': 42, 'bar': 37}['foo'] + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB19287512(self): + unformatted_code = textwrap.dedent("""\ + class Foo(object): + + def bar(self): + with xxxxxxxxxx.yyyyy( + 'aaaaaaa.bbbbbbbb.ccccccc.dddddddddddddddddddd.eeeeeeeeeee', + fffffffffff=(aaaaaaa.bbbbbbbb.ccccccc.dddddddddddddddddddd + .Mmmmmmmmmmmmmmmmmm(-1, 'permission error'))): + self.assertRaises(nnnnnnnnnnnnnnnn.ooooo, ppppp.qqqqqqqqqqqqqqqqq) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + class Foo(object): + + def bar(self): + with xxxxxxxxxx.yyyyy( + 'aaaaaaa.bbbbbbbb.ccccccc.dddddddddddddddddddd.eeeeeeeeeee', + fffffffffff=( + aaaaaaa.bbbbbbbb.ccccccc.dddddddddddddddddddd.Mmmmmmmmmmmmmmmmmm( + -1, 'permission error'))): + self.assertRaises(nnnnnnnnnnnnnnnn.ooooo, ppppp.qqqqqqqqqqqqqqqqq) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB19194420(self): + code = textwrap.dedent("""\ + method.Set( + 'long argument goes here that causes the line to break', + lambda arg2=0.5: arg2) + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB19073499(self): + code = textwrap.dedent("""\ + instance = ( + aaaaaaa.bbbbbbb().ccccccccccccccccc().ddddddddddd({ + 'aa': 'context!' + }).eeeeeeeeeeeeeeeeeee({ # Inline comment about why fnord has the value 6. + 'fnord': 6 + })) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB18257115(self): + code = textwrap.dedent("""\ + if True: + if True: + self._Test(aaaa, bbbbbbb.cccccccccc, dddddddd, eeeeeeeeeee, + [ffff, ggggggggggg, hhhhhhhhhhhh, iiiiii, jjjj]) + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB18256666(self): + code = textwrap.dedent("""\ + class Foo(object): + + def Bar(self): + aaaaa.bbbbbbb( + ccc='ddddddddddddddd', + eeee='ffffffffffffffffffffff-%s-%s' % (gggg, int(time.time())), + hhhhhh={ + 'iiiiiiiiiii': iiiiiiiiiii, + 'jjjj': jjjj.jjjjj(), + 'kkkkkkkkkkkk': kkkkkkkkkkkk, + }, + llllllllll=mmmmmm.nnnnnnnnnnnnnnnn) + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB18256826(self): + code = textwrap.dedent("""\ + if True: + pass + # A multiline comment. + # Line two. + elif False: + pass + + if True: + pass + # A multiline comment. + # Line two. + elif False: + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB18255697(self): + code = textwrap.dedent("""\ + AAAAAAAAAAAAAAA = { + 'XXXXXXXXXXXXXX': 4242, # Inline comment + # Next comment + 'YYYYYYYYYYYYYYYY': ['zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz'], + } + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testB17534869(self): + unformatted_code = textwrap.dedent("""\ + if True: + self.assertLess(abs(time.time()-aaaa.bbbbbbbbbbb( + datetime.datetime.now())), 1) + """) + expected_formatted_code = textwrap.dedent("""\ + if True: + self.assertLess( + abs(time.time() - aaaa.bbbbbbbbbbb(datetime.datetime.now())), 1) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB17489866(self): + unformatted_code = textwrap.dedent("""\ + def f(): + if True: + if True: + return aaaa.bbbbbbbbb(ccccccc=dddddddddddddd({('eeee', 'ffffffff'): str(j)})) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + def f(): + if True: + if True: + return aaaa.bbbbbbbbb( + ccccccc=dddddddddddddd({('eeee', 'ffffffff'): str(j)})) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB17133019(self): + unformatted_code = textwrap.dedent("""\ + class aaaaaaaaaaaaaa(object): + + def bbbbbbbbbb(self): + with io.open("/dev/null", "rb"): + with io.open(os.path.join(aaaaa.bbbbb.ccccccccccc, + DDDDDDDDDDDDDDD, + "eeeeeeeee ffffffffff" + ), "rb") as gggggggggggggggggggg: + print(gggggggggggggggggggg) + """) + expected_formatted_code = textwrap.dedent("""\ + class aaaaaaaaaaaaaa(object): + + def bbbbbbbbbb(self): + with io.open("/dev/null", "rb"): + with io.open( + os.path.join(aaaaa.bbbbb.ccccccccccc, DDDDDDDDDDDDDDD, + "eeeeeeeee ffffffffff"), "rb") as gggggggggggggggggggg: + print(gggggggggggggggggggg) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB17011869(self): + unformatted_code = textwrap.dedent("""\ + '''blah......''' + + class SomeClass(object): + '''blah.''' + + AAAAAAAAAAAA = { # Comment. + 'BBB': 1.0, + 'DDDDDDDD': 0.4811 + } + """) + expected_formatted_code = textwrap.dedent("""\ + '''blah......''' + + + class SomeClass(object): + '''blah.''' + + AAAAAAAAAAAA = { # Comment. + 'BBB': 1.0, + 'DDDDDDDD': 0.4811 + } + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB16783631(self): + unformatted_code = textwrap.dedent("""\ + if True: + with aaaaaaaaaaaaaa.bbbbbbbbbbbbb.ccccccc(ddddddddddddd, + eeeeeeeee=self.fffffffffffff + )as gggg: + pass + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + if True: + with aaaaaaaaaaaaaa.bbbbbbbbbbbbb.ccccccc( + ddddddddddddd, eeeeeeeee=self.fffffffffffff) as gggg: + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB16572361(self): + unformatted_code = textwrap.dedent("""\ + def foo(self): + def bar(my_dict_name): + self.my_dict_name['foo-bar-baz-biz-boo-baa-baa'].IncrementBy.assert_called_once_with('foo_bar_baz_boo') + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + def foo(self): + + def bar(my_dict_name): + self.my_dict_name[ + 'foo-bar-baz-biz-boo-baa-baa'].IncrementBy.assert_called_once_with( + 'foo_bar_baz_boo') + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB15884241(self): + unformatted_code = textwrap.dedent("""\ + if 1: + if 1: + for row in AAAA: + self.create(aaaaaaaa="/aaa/bbbb/cccc/dddddd/eeeeeeeeeeeeeeeeeeeeeeeeee/%s" % row [0].replace(".foo", ".bar"), aaaaa=bbb[1], ccccc=bbb[2], dddd=bbb[3], eeeeeeeeeee=[s.strip() for s in bbb[4].split(",")], ffffffff=[s.strip() for s in bbb[5].split(",")], gggggg=bbb[6]) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + if 1: + if 1: + for row in AAAA: + self.create( + aaaaaaaa="/aaa/bbbb/cccc/dddddd/eeeeeeeeeeeeeeeeeeeeeeeeee/%s" % + row[0].replace(".foo", ".bar"), + aaaaa=bbb[1], + ccccc=bbb[2], + dddd=bbb[3], + eeeeeeeeeee=[s.strip() for s in bbb[4].split(",")], + ffffffff=[s.strip() for s in bbb[5].split(",")], + gggggg=bbb[6]) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB15697268(self): + unformatted_code = textwrap.dedent("""\ + def main(unused_argv): + ARBITRARY_CONSTANT_A = 10 + an_array_with_an_exceedingly_long_name = range(ARBITRARY_CONSTANT_A + 1) + ok = an_array_with_an_exceedingly_long_name[:ARBITRARY_CONSTANT_A] + bad_slice = map(math.sqrt, an_array_with_an_exceedingly_long_name[:ARBITRARY_CONSTANT_A]) + a_long_name_slicing = an_array_with_an_exceedingly_long_name[:ARBITRARY_CONSTANT_A] + bad_slice = ("I am a crazy, no good, string what's too long, etc." + " no really ")[:ARBITRARY_CONSTANT_A] + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + def main(unused_argv): + ARBITRARY_CONSTANT_A = 10 + an_array_with_an_exceedingly_long_name = range(ARBITRARY_CONSTANT_A + 1) + ok = an_array_with_an_exceedingly_long_name[:ARBITRARY_CONSTANT_A] + bad_slice = map(math.sqrt, + an_array_with_an_exceedingly_long_name[:ARBITRARY_CONSTANT_A]) + a_long_name_slicing = an_array_with_an_exceedingly_long_name[: + ARBITRARY_CONSTANT_A] + bad_slice = ("I am a crazy, no good, string what's too long, etc." + + " no really ")[:ARBITRARY_CONSTANT_A] + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB15597568(self): + unformatted_code = textwrap.dedent("""\ + if True: + if True: + if True: + print(("Return code was %d" + (", and the process timed out." if did_time_out else ".")) % errorcode) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + if True: + if True: + if True: + print(("Return code was %d" + + (", and the process timed out." if did_time_out else ".")) % + errorcode) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB15542157(self): + unformatted_code = textwrap.dedent("""\ + aaaaaaaaaaaa = bbbb.ccccccccccccccc(dddddd.eeeeeeeeeeeeee, ffffffffffffffffff, gggggg.hhhhhhhhhhhhhhhhh) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + aaaaaaaaaaaa = bbbb.ccccccccccccccc(dddddd.eeeeeeeeeeeeee, ffffffffffffffffff, + gggggg.hhhhhhhhhhhhhhhhh) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB15438132(self): + unformatted_code = textwrap.dedent("""\ + if aaaaaaa.bbbbbbbbbb: + cccccc.dddddddddd(eeeeeeeeeee=fffffffffffff.gggggggggggggggggg) + if hhhhhh.iiiii.jjjjjjjjjjjjj: + # This is a comment in the middle of it all. + kkkkkkk.llllllllll.mmmmmmmmmmmmm = True + if (aaaaaa.bbbbb.ccccccccccccc != ddddddd.eeeeeeeeee.fffffffffffff or + eeeeee.fffff.ggggggggggggggggggggggggggg() != hhhhhhh.iiiiiiiiii.jjjjjjjjjjjj): + aaaaaaaa.bbbbbbbbbbbb( + aaaaaa.bbbbb.cc, + dddddddddddd=eeeeeeeeeeeeeeeeeee.fffffffffffffffff( + gggggg.hh, + iiiiiiiiiiiiiiiiiii.jjjjjjjjjj.kkkkkkk, + lllll.mm), + nnnnnnnnnn=ooooooo.pppppppppp) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + if aaaaaaa.bbbbbbbbbb: + cccccc.dddddddddd(eeeeeeeeeee=fffffffffffff.gggggggggggggggggg) + if hhhhhh.iiiii.jjjjjjjjjjjjj: + # This is a comment in the middle of it all. + kkkkkkk.llllllllll.mmmmmmmmmmmmm = True + if (aaaaaa.bbbbb.ccccccccccccc != ddddddd.eeeeeeeeee.fffffffffffff or + eeeeee.fffff.ggggggggggggggggggggggggggg() + != hhhhhhh.iiiiiiiiii.jjjjjjjjjjjj): + aaaaaaaa.bbbbbbbbbbbb( + aaaaaa.bbbbb.cc, + dddddddddddd=eeeeeeeeeeeeeeeeeee.fffffffffffffffff( + gggggg.hh, iiiiiiiiiiiiiiiiiii.jjjjjjjjjj.kkkkkkk, lllll.mm), + nnnnnnnnnn=ooooooo.pppppppppp) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB14468247(self): + unformatted_code = textwrap.dedent("""\ + call(a=1, + b=2, + ) + """) + expected_formatted_code = textwrap.dedent("""\ + call( + a=1, + b=2, + ) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB14406499(self): + unformatted_code = textwrap.dedent("""\ + def foo1(parameter_1, parameter_2, parameter_3, parameter_4, \ +parameter_5, parameter_6): pass + """) + expected_formatted_code = textwrap.dedent("""\ + def foo1(parameter_1, parameter_2, parameter_3, parameter_4, parameter_5, + parameter_6): + pass + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB13900309(self): + unformatted_code = textwrap.dedent("""\ + self.aaaaaaaaaaa( # A comment in the middle of it all. + 948.0/3600, self.bbb.ccccccccccccccccccccc(dddddddddddddddd.eeee, True)) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + self.aaaaaaaaaaa( # A comment in the middle of it all. + 948.0 / 3600, self.bbb.ccccccccccccccccccccc(dddddddddddddddd.eeee, True)) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + code = textwrap.dedent("""\ + aaaaaaaaaa.bbbbbbbbbbbbbbbbbbbbbbbb.cccccccccccccccccccccccccccccc( + DC_1, (CL - 50, CL), AAAAAAAA, BBBBBBBBBBBBBBBB, 98.0, + CCCCCCC).ddddddddd( # Look! A comment is here. + AAAAAAAA - (20 * 60 - 5)) + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + unformatted_code = textwrap.dedent("""\ + aaaaaaaaaaaaaaaaaaaaaaaa.bbbbbbbbbbbbb.ccccccccccccccccccccccccc().dddddddddddddddddddddddddd(1, 2, 3, 4) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + aaaaaaaaaaaaaaaaaaaaaaaa.bbbbbbbbbbbbb.ccccccccccccccccccccccccc( + ).dddddddddddddddddddddddddd(1, 2, 3, 4) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + unformatted_code = textwrap.dedent("""\ + aaaaaaaaaaaaaaaaaaaaaaaa.bbbbbbbbbbbbb.ccccccccccccccccccccccccc(x).dddddddddddddddddddddddddd(1, 2, 3, 4) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + aaaaaaaaaaaaaaaaaaaaaaaa.bbbbbbbbbbbbb.ccccccccccccccccccccccccc( + x).dddddddddddddddddddddddddd(1, 2, 3, 4) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + unformatted_code = textwrap.dedent("""\ + aaaaaaaaaaaaaaaaaaaaaaaa(xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx).dddddddddddddddddddddddddd(1, 2, 3, 4) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + aaaaaaaaaaaaaaaaaaaaaaaa( + xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx).dddddddddddddddddddddddddd(1, 2, 3, 4) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + unformatted_code = textwrap.dedent("""\ + aaaaaaaaaaaaaaaaaaaaaaaa().bbbbbbbbbbbbbbbbbbbbbbbb().ccccccccccccccccccc().\ +dddddddddddddddddd().eeeeeeeeeeeeeeeeeeeee().fffffffffffffffff().gggggggggggggggggg() + """) + expected_formatted_code = textwrap.dedent("""\ + aaaaaaaaaaaaaaaaaaaaaaaa().bbbbbbbbbbbbbbbbbbbbbbbb().ccccccccccccccccccc( + ).dddddddddddddddddd().eeeeeeeeeeeeeeeeeeeee().fffffffffffffffff( + ).gggggggggggggggggg() + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testB67935687(self): + code = textwrap.dedent("""\ + Fetch( + Raw('monarch.BorgTask', '/union/row_operator_action_delay'), + {'borg_user': self.borg_user}) + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + unformatted_code = textwrap.dedent("""\ + shelf_renderer.expand_text = text.translate_to_unicode( + expand_text % { + 'creator': creator + }) + """) + expected_formatted_code = textwrap.dedent("""\ + shelf_renderer.expand_text = text.translate_to_unicode(expand_text % + {'creator': creator}) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + +if __name__ == '__main__': + unittest.main() diff --git a/yapftests/reformatter_facebook_test.py b/yapftests/reformatter_facebook_test.py new file mode 100644 index 000000000..dfb87d378 --- /dev/null +++ b/yapftests/reformatter_facebook_test.py @@ -0,0 +1,431 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Facebook tests for yapf.reformatter.""" + +import textwrap +import unittest + +from yapf.yapflib import reformatter +from yapf.yapflib import style + +from yapftests import yapf_test_helper + + +class TestsForFacebookStyle(yapf_test_helper.YAPFTest): + + @classmethod + def setUpClass(cls): + style.SetGlobalStyle(style.CreateFacebookStyle()) + + def testNoNeedForLineBreaks(self): + unformatted_code = textwrap.dedent("""\ + def overly_long_function_name( + just_one_arg, **kwargs): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + def overly_long_function_name(just_one_arg, **kwargs): + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testDedentClosingBracket(self): + unformatted_code = textwrap.dedent("""\ + def overly_long_function_name( + first_argument_on_the_same_line, + second_argument_makes_the_line_too_long): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + def overly_long_function_name( + first_argument_on_the_same_line, second_argument_makes_the_line_too_long + ): + pass + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testBreakAfterOpeningBracketIfContentsTooBig(self): + unformatted_code = textwrap.dedent("""\ + def overly_long_function_name(a, b, c, d, e, f, g, h, i, j, k, l, m, + n, o, p, q, r, s, t, u, v, w, x, y, z): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + def overly_long_function_name( + a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x, y, z + ): + pass + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testDedentClosingBracketWithComments(self): + unformatted_code = textwrap.dedent("""\ + def overly_long_function_name( + # comment about the first argument + first_argument_with_a_very_long_name_or_so, + # comment about the second argument + second_argument_makes_the_line_too_long): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + def overly_long_function_name( + # comment about the first argument + first_argument_with_a_very_long_name_or_so, + # comment about the second argument + second_argument_makes_the_line_too_long + ): + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testDedentImportAsNames(self): + code = textwrap.dedent("""\ + from module import ( + internal_function as function, + SOME_CONSTANT_NUMBER1, + SOME_CONSTANT_NUMBER2, + SOME_CONSTANT_NUMBER3, + ) + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testDedentTestListGexp(self): + unformatted_code = textwrap.dedent("""\ + try: + pass + except ( + IOError, OSError, LookupError, RuntimeError, OverflowError + ) as exception: + pass + + try: + pass + except ( + IOError, OSError, LookupError, RuntimeError, OverflowError, + ) as exception: + pass + """) + expected_formatted_code = textwrap.dedent("""\ + try: + pass + except ( + IOError, OSError, LookupError, RuntimeError, OverflowError + ) as exception: + pass + + try: + pass + except ( + IOError, + OSError, + LookupError, + RuntimeError, + OverflowError, + ) as exception: + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testBrokenIdempotency(self): + # TODO(ambv): The following behaviour should be fixed. + pass0_code = textwrap.dedent("""\ + try: + pass + except (IOError, OSError, LookupError, RuntimeError, OverflowError) as exception: + pass + """) # noqa + pass1_code = textwrap.dedent("""\ + try: + pass + except ( + IOError, OSError, LookupError, RuntimeError, OverflowError + ) as exception: + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(pass0_code) + self.assertCodeEqual(pass1_code, reformatter.Reformat(llines)) + + pass2_code = textwrap.dedent("""\ + try: + pass + except ( + IOError, OSError, LookupError, RuntimeError, OverflowError + ) as exception: + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(pass1_code) + self.assertCodeEqual(pass2_code, reformatter.Reformat(llines)) + + def testIfExprHangingIndent(self): + unformatted_code = textwrap.dedent("""\ + if True: + if True: + if True: + if not self.frobbies and ( + self.foobars.counters['db.cheeses'] != 1 or + self.foobars.counters['db.marshmellow_skins'] != 1): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + if True: + if True: + if True: + if not self.frobbies and ( + self.foobars.counters['db.cheeses'] != 1 or + self.foobars.counters['db.marshmellow_skins'] != 1 + ): + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testSimpleDedenting(self): + unformatted_code = textwrap.dedent("""\ + if True: + self.assertEqual(result.reason_not_added, "current preflight is still running") + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + if True: + self.assertEqual( + result.reason_not_added, "current preflight is still running" + ) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testDedentingWithSubscripts(self): + unformatted_code = textwrap.dedent("""\ + class Foo: + class Bar: + @classmethod + def baz(cls, clues_list, effect, constraints, constraint_manager): + if clues_lists: + return cls.single_constraint_not(clues_lists, effect, constraints[0], constraint_manager) + + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + class Foo: + class Bar: + @classmethod + def baz(cls, clues_list, effect, constraints, constraint_manager): + if clues_lists: + return cls.single_constraint_not( + clues_lists, effect, constraints[0], constraint_manager + ) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testDedentingCallsWithInnerLists(self): + code = textwrap.dedent("""\ + class _(): + def _(): + cls.effect_clues = { + 'effect': Clue((cls.effect_time, 'apache_host'), effect_line, 40) + } + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testDedentingListComprehension(self): + unformatted_code = textwrap.dedent("""\ + class Foo(): + def _pack_results_for_constraint_or(): + self.param_groups = dict( + ( + key + 1, ParamGroup(groups[key], default_converter) + ) for key in six.moves.range(len(groups)) + ) + + for combination in cls._clues_combinations(clues_lists): + if all( + cls._verify_constraint(combination, effect, constraint) + for constraint in constraints + ): + pass + + guessed_dict = dict( + ( + key, guessed_pattern_matches[key] + ) for key in six.moves.range(len(guessed_pattern_matches)) + ) + + content = "".join( + itertools.chain( + (first_line_fragment, ), lines_between, (last_line_fragment, ) + ) + ) + + rule = Rule( + [self.cause1, self.cause2, self.cause1, self.cause2], self.effect, constraints1, + Rule.LINKAGE_AND + ) + + assert sorted(log_type.files_to_parse) == [ + ('localhost', os.path.join(path, 'node_1.log'), super_parser), + ('localhost', os.path.join(path, 'node_2.log'), super_parser) + ] + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + class Foo(): + def _pack_results_for_constraint_or(): + self.param_groups = dict( + (key + 1, ParamGroup(groups[key], default_converter)) + for key in six.moves.range(len(groups)) + ) + + for combination in cls._clues_combinations(clues_lists): + if all( + cls._verify_constraint(combination, effect, constraint) + for constraint in constraints + ): + pass + + guessed_dict = dict( + (key, guessed_pattern_matches[key]) + for key in six.moves.range(len(guessed_pattern_matches)) + ) + + content = "".join( + itertools.chain( + (first_line_fragment, ), lines_between, (last_line_fragment, ) + ) + ) + + rule = Rule( + [self.cause1, self.cause2, self.cause1, self.cause2], self.effect, + constraints1, Rule.LINKAGE_AND + ) + + assert sorted(log_type.files_to_parse) == [ + ('localhost', os.path.join(path, 'node_1.log'), super_parser), + ('localhost', os.path.join(path, 'node_2.log'), super_parser) + ] + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testMustSplitDedenting(self): + code = textwrap.dedent("""\ + class _(): + def _(): + effect_line = FrontInput( + effect_line_offset, line_content, + LineSource('localhost', xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx) + ) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testDedentIfConditional(self): + code = textwrap.dedent("""\ + class _(): + def _(): + if True: + if not self.frobbies and ( + self.foobars.counters['db.cheeses'] != 1 or + self.foobars.counters['db.marshmellow_skins'] != 1 + ): + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testDedentSet(self): + code = textwrap.dedent("""\ + class _(): + def _(): + assert set(self.constraint_links.get_links()) == set( + [ + (2, 10, 100), + (2, 10, 200), + (2, 20, 100), + (2, 20, 200), + ] + ) + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testDedentingInnerScope(self): + code = textwrap.dedent("""\ + class Foo(): + @classmethod + def _pack_results_for_constraint_or(cls, combination, constraints): + return cls._create_investigation_result( + (clue for clue in combination if not clue == Verifier.UNMATCHED), + constraints, InvestigationResult.OR + ) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + reformatted_code = reformatter.Reformat(llines) + self.assertCodeEqual(code, reformatted_code) + + llines = yapf_test_helper.ParseAndUnwrap(reformatted_code) + reformatted_code = reformatter.Reformat(llines) + self.assertCodeEqual(code, reformatted_code) + + def testCommentWithNewlinesInPrefix(self): + unformatted_code = textwrap.dedent("""\ + def foo(): + if 0: + return False + + + #a deadly comment + elif 1: + return True + + + print(foo()) + """) + expected_formatted_code = textwrap.dedent("""\ + def foo(): + if 0: + return False + + #a deadly comment + elif 1: + return True + + + print(foo()) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testIfStmtClosingBracket(self): + unformatted_code = textwrap.dedent("""\ + if (isinstance(value , (StopIteration , StopAsyncIteration )) and exc.__cause__ is value_asdfasdfasdfasdfsafsafsafdasfasdfs): + return False + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + if ( + isinstance(value, (StopIteration, StopAsyncIteration)) and + exc.__cause__ is value_asdfasdfasdfasdfsafsafsafdasfasdfs + ): + return False + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + +if __name__ == '__main__': + unittest.main() diff --git a/yapftests/reformatter_pep8_test.py b/yapftests/reformatter_pep8_test.py new file mode 100644 index 000000000..1cf7820e2 --- /dev/null +++ b/yapftests/reformatter_pep8_test.py @@ -0,0 +1,870 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PEP8 tests for yapf.reformatter.""" + +import textwrap +import unittest + +from yapf.yapflib import reformatter +from yapf.yapflib import style + +from yapftests import yapf_test_helper + + +class TestsForPEP8Style(yapf_test_helper.YAPFTest): + + @classmethod + def setUpClass(cls): # pylint: disable=g-missing-super-call + style.SetGlobalStyle(style.CreatePEP8Style()) + + def testIndent4(self): + unformatted_code = textwrap.dedent("""\ + if a+b: + pass + """) + expected_formatted_code = textwrap.dedent("""\ + if a + b: + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testSingleLineIfStatements(self): + code = textwrap.dedent("""\ + if True: a = 42 + elif False: b = 42 + else: c = 42 + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testBlankBetweenClassAndDef(self): + unformatted_code = textwrap.dedent("""\ + class Foo: + def joe(): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + class Foo: + + def joe(): + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testBlankBetweenDefsInClass(self): + unformatted_code = textwrap.dedent('''\ + class TestClass: + def __init__(self): + self.running = False + def run(self): + """Override in subclass""" + def is_running(self): + return self.running + ''') + expected_formatted_code = textwrap.dedent('''\ + class TestClass: + + def __init__(self): + self.running = False + + def run(self): + """Override in subclass""" + + def is_running(self): + return self.running + ''') + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testSingleWhiteBeforeTrailingComment(self): + unformatted_code = textwrap.dedent("""\ + if a+b: # comment + pass + """) + expected_formatted_code = textwrap.dedent("""\ + if a + b: # comment + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testSpaceBetweenEndingCommandAndClosingBracket(self): + unformatted_code = textwrap.dedent("""\ + a = ( + 1, + ) + """) + expected_formatted_code = textwrap.dedent("""\ + a = (1, ) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testContinuedNonOutdentedLine(self): + code = textwrap.dedent("""\ + class eld(d): + if str(geom.geom_type).upper( + ) != self.geom_type and not self.geom_type == 'GEOMETRY': + ror(code='om_type') + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testWrappingPercentExpressions(self): + unformatted_code = textwrap.dedent("""\ + def f(): + if True: + zzzzz = '%s-%s' % (xxxxxxxxxxxxxxxxxxxxxxxxxx + 1, xxxxxxxxxxxxxxxxx.yyy + 1) + zzzzz = '%s-%s'.ww(xxxxxxxxxxxxxxxxxxxxxxxxxx + 1, xxxxxxxxxxxxxxxxx.yyy + 1) + zzzzz = '%s-%s' % (xxxxxxxxxxxxxxxxxxxxxxx + 1, xxxxxxxxxxxxxxxxxxxxx + 1) + zzzzz = '%s-%s'.ww(xxxxxxxxxxxxxxxxxxxxxxx + 1, xxxxxxxxxxxxxxxxxxxxx + 1) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + def f(): + if True: + zzzzz = '%s-%s' % (xxxxxxxxxxxxxxxxxxxxxxxxxx + 1, + xxxxxxxxxxxxxxxxx.yyy + 1) + zzzzz = '%s-%s'.ww(xxxxxxxxxxxxxxxxxxxxxxxxxx + 1, + xxxxxxxxxxxxxxxxx.yyy + 1) + zzzzz = '%s-%s' % (xxxxxxxxxxxxxxxxxxxxxxx + 1, + xxxxxxxxxxxxxxxxxxxxx + 1) + zzzzz = '%s-%s'.ww(xxxxxxxxxxxxxxxxxxxxxxx + 1, + xxxxxxxxxxxxxxxxxxxxx + 1) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testAlignClosingBracketWithVisualIndentation(self): + unformatted_code = textwrap.dedent("""\ + TEST_LIST = ('foo', 'bar', # first comment + 'baz' # second comment + ) + """) + expected_formatted_code = textwrap.dedent("""\ + TEST_LIST = ( + 'foo', + 'bar', # first comment + 'baz' # second comment + ) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + unformatted_code = textwrap.dedent("""\ + def f(): + + def g(): + while (xxxxxxxxxxxxxxxxxxxx(yyyyyyyyyyyyy[zzzzz]) == 'aaaaaaaaaaa' and + xxxxxxxxxxxxxxxxxxxx(yyyyyyyyyyyyy[zzzzz].aaaaaaaa[0]) == 'bbbbbbb' + ): + pass + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + def f(): + + def g(): + while (xxxxxxxxxxxxxxxxxxxx(yyyyyyyyyyyyy[zzzzz]) == 'aaaaaaaaaaa' + and xxxxxxxxxxxxxxxxxxxx( + yyyyyyyyyyyyy[zzzzz].aaaaaaaa[0]) == 'bbbbbbb'): + pass + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testIndentSizeChanging(self): + unformatted_code = textwrap.dedent("""\ + if True: + runtime_mins = (program_end_time - program_start_time).total_seconds() / 60.0 + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + if True: + runtime_mins = (program_end_time - + program_start_time).total_seconds() / 60.0 + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testHangingIndentCollision(self): + unformatted_code = textwrap.dedent("""\ + if (aaaaaaaaaaaaaa + bbbbbbbbbbbbbbbb == ccccccccccccccccc and xxxxxxxxxxxxx or yyyyyyyyyyyyyyyyy): + pass + elif (xxxxxxxxxxxxxxx(aaaaaaaaaaa, bbbbbbbbbbbbbb, cccccccccccc, dddddddddd=None)): + pass + + + def h(): + if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): + pass + + for connection in itertools.chain(branch.contact, branch.address, morestuff.andmore.andmore.andmore.andmore.andmore.andmore.andmore): + dosomething(connection) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + if (aaaaaaaaaaaaaa + bbbbbbbbbbbbbbbb == ccccccccccccccccc and xxxxxxxxxxxxx + or yyyyyyyyyyyyyyyyy): + pass + elif (xxxxxxxxxxxxxxx(aaaaaaaaaaa, + bbbbbbbbbbbbbb, + cccccccccccc, + dddddddddd=None)): + pass + + + def h(): + if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and + xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): + pass + + for connection in itertools.chain( + branch.contact, branch.address, + morestuff.andmore.andmore.andmore.andmore.andmore.andmore.andmore): + dosomething(connection) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testSplittingBeforeLogicalOperator(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{based_on_style: pep8, split_before_logical_operator: True}')) + unformatted_code = textwrap.dedent("""\ + def foo(): + return bool(update.message.new_chat_member or update.message.left_chat_member or + update.message.new_chat_title or update.message.new_chat_photo or + update.message.delete_chat_photo or update.message.group_chat_created or + update.message.supergroup_chat_created or update.message.channel_chat_created + or update.message.migrate_to_chat_id or update.message.migrate_from_chat_id or + update.message.pinned_message) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + def foo(): + return bool( + update.message.new_chat_member or update.message.left_chat_member + or update.message.new_chat_title or update.message.new_chat_photo + or update.message.delete_chat_photo + or update.message.group_chat_created + or update.message.supergroup_chat_created + or update.message.channel_chat_created + or update.message.migrate_to_chat_id + or update.message.migrate_from_chat_id + or update.message.pinned_message) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreatePEP8Style()) + + def testContiguousListEndingWithComment(self): + unformatted_code = textwrap.dedent("""\ + if True: + if True: + keys.append(aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa) # may be unassigned. + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + if True: + if True: + keys.append( + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa) # may be unassigned. + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testSplittingBeforeFirstArgument(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{based_on_style: pep8, split_before_first_argument: True}')) + unformatted_code = textwrap.dedent("""\ + a_very_long_function_name(long_argument_name_1=1, long_argument_name_2=2, + long_argument_name_3=3, long_argument_name_4=4) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + a_very_long_function_name( + long_argument_name_1=1, + long_argument_name_2=2, + long_argument_name_3=3, + long_argument_name_4=4) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreatePEP8Style()) + + def testSplittingExpressionsInsideSubscripts(self): + unformatted_code = textwrap.dedent("""\ + def foo(): + df = df[(df['campaign_status'] == 'LIVE') & (df['action_status'] == 'LIVE')] + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + def foo(): + df = df[(df['campaign_status'] == 'LIVE') + & (df['action_status'] == 'LIVE')] + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testSplitListsAndDictSetMakersIfCommaTerminated(self): + unformatted_code = textwrap.dedent("""\ + DJANGO_TEMPLATES_OPTIONS = {"context_processors": []} + DJANGO_TEMPLATES_OPTIONS = {"context_processors": [],} + x = ["context_processors"] + x = ["context_processors",] + """) + expected_formatted_code = textwrap.dedent("""\ + DJANGO_TEMPLATES_OPTIONS = {"context_processors": []} + DJANGO_TEMPLATES_OPTIONS = { + "context_processors": [], + } + x = ["context_processors"] + x = [ + "context_processors", + ] + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testSplitAroundNamedAssigns(self): + unformatted_code = textwrap.dedent("""\ + class a(): + + def a(): return a( + aaaaaaaaaa=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + class a(): + + def a(): + return a( + aaaaaaaaaa=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + ) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testUnaryOperator(self): + unformatted_code = textwrap.dedent("""\ + if not -3 < x < 3: + pass + if -3 < x < 3: + pass + """) + expected_formatted_code = textwrap.dedent("""\ + if not -3 < x < 3: + pass + if -3 < x < 3: + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testNoSplitBeforeDictValue(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig('{based_on_style: pep8, ' + 'allow_split_before_dict_value: false, ' + 'coalesce_brackets: true, ' + 'dedent_closing_brackets: true, ' + 'each_dict_entry_on_separate_line: true, ' + 'split_before_logical_operator: true}')) + + unformatted_code = textwrap.dedent("""\ + some_dict = { + 'title': _("I am example data"), + 'description': _("Lorem ipsum dolor met sit amet elit, si vis pacem para bellum " + "elites nihi very long string."), + } + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + some_dict = { + 'title': _("I am example data"), + 'description': _( + "Lorem ipsum dolor met sit amet elit, si vis pacem para bellum " + "elites nihi very long string." + ), + } + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + + unformatted_code = textwrap.dedent("""\ + X = {'a': 1, 'b': 2, 'key': this_is_a_function_call_that_goes_over_the_column_limit_im_pretty_sure()} + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + X = { + 'a': 1, + 'b': 2, + 'key': this_is_a_function_call_that_goes_over_the_column_limit_im_pretty_sure() + } + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + + unformatted_code = textwrap.dedent("""\ + attrs = { + 'category': category, + 'role': forms.ModelChoiceField(label=_("Role"), required=False, queryset=category_roles, initial=selected_role, empty_label=_("No access"),), + } + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + attrs = { + 'category': category, + 'role': forms.ModelChoiceField( + label=_("Role"), + required=False, + queryset=category_roles, + initial=selected_role, + empty_label=_("No access"), + ), + } + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + + unformatted_code = textwrap.dedent("""\ + css_class = forms.CharField( + label=_("CSS class"), + required=False, + help_text=_("Optional CSS class used to customize this category appearance from templates."), + ) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + css_class = forms.CharField( + label=_("CSS class"), + required=False, + help_text=_( + "Optional CSS class used to customize this category appearance from templates." + ), + ) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreatePEP8Style()) + + def testBitwiseOperandSplitting(self): + unformatted_code = textwrap.dedent("""\ + def _(): + include_values = np.where( + (cdffile['Quality_Flag'][:] >= 5) & ( + cdffile['Day_Night_Flag'][:] == 1) & ( + cdffile['Longitude'][:] >= select_lon - radius) & ( + cdffile['Longitude'][:] <= select_lon + radius) & ( + cdffile['Latitude'][:] >= select_lat - radius) & ( + cdffile['Latitude'][:] <= select_lat + radius)) + """) # noqa + expected_code = textwrap.dedent("""\ + def _(): + include_values = np.where( + (cdffile['Quality_Flag'][:] >= 5) & (cdffile['Day_Night_Flag'][:] == 1) + & (cdffile['Longitude'][:] >= select_lon - radius) + & (cdffile['Longitude'][:] <= select_lon + radius) + & (cdffile['Latitude'][:] >= select_lat - radius) + & (cdffile['Latitude'][:] <= select_lat + radius)) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertEqual(expected_code, reformatter.Reformat(llines)) + + def testNoBlankLinesOnlyForFirstNestedObject(self): + unformatted_code = textwrap.dedent('''\ + class Demo: + """ + Demo docs + """ + def foo(self): + """ + foo docs + """ + def bar(self): + """ + bar docs + """ + ''') + expected_code = textwrap.dedent('''\ + class Demo: + """ + Demo docs + """ + + def foo(self): + """ + foo docs + """ + + def bar(self): + """ + bar docs + """ + ''') + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertEqual(expected_code, reformatter.Reformat(llines)) + + def testSplitBeforeArithmeticOperators(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{based_on_style: pep8, split_before_arithmetic_operator: true}')) + + unformatted_code = textwrap.dedent("""\ + def _(): + raise ValueError('This is a long message that ends with an argument: ' + str(42)) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + def _(): + raise ValueError('This is a long message that ends with an argument: ' + + str(42)) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreatePEP8Style()) + + def testListSplitting(self): + unformatted_code = textwrap.dedent("""\ + foo([(1,1), (1,1), (1,1), (1,1), (1,1), (1,1), (1,1), + (1,1), (1,1), (1,1), (1,1), (1,1), (1,1), (1,1), + (1,10), (1,11), (1, 10), (1,11), (10,11)]) + """) + expected_code = textwrap.dedent("""\ + foo([(1, 1), (1, 1), (1, 1), (1, 1), (1, 1), (1, 1), (1, 1), (1, 1), (1, 1), + (1, 1), (1, 1), (1, 1), (1, 1), (1, 1), (1, 10), (1, 11), (1, 10), + (1, 11), (10, 11)]) + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_code, reformatter.Reformat(llines)) + + def testNoBlankLineBeforeNestedFuncOrClass(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{based_on_style: pep8, ' + 'blank_line_before_nested_class_or_def: false}')) + + unformatted_code = textwrap.dedent('''\ + def normal_function(): + """Return the nested function.""" + + def nested_function(): + """Do nothing just nest within.""" + + @nested(klass) + class nested_class(): + pass + + pass + + return nested_function + ''') + expected_formatted_code = textwrap.dedent('''\ + def normal_function(): + """Return the nested function.""" + def nested_function(): + """Do nothing just nest within.""" + @nested(klass) + class nested_class(): + pass + + pass + + return nested_function + ''') + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreatePEP8Style()) + + def testParamListIndentationCollision1(self): + unformatted_code = textwrap.dedent("""\ + class _(): + + def __init__(self, title: Optional[str], diffs: Collection[BinaryDiff] = (), charset: Union[Type[AsciiCharset], Type[LineCharset]] = AsciiCharset, preprocess: Callable[[str], str] = identity, + # TODO(somebody): Make this a Literal type. + justify: str = 'rjust'): + self._cs = charset + self._preprocess = preprocess + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + class _(): + + def __init__( + self, + title: Optional[str], + diffs: Collection[BinaryDiff] = (), + charset: Union[Type[AsciiCharset], + Type[LineCharset]] = AsciiCharset, + preprocess: Callable[[str], str] = identity, + # TODO(somebody): Make this a Literal type. + justify: str = 'rjust'): + self._cs = charset + self._preprocess = preprocess + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testParamListIndentationCollision2(self): + code = textwrap.dedent("""\ + def simple_pass_function_with_an_extremely_long_name_and_some_arguments( + argument0, argument1): + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testParamListIndentationCollision3(self): + code = textwrap.dedent("""\ + def func1( + arg1, + arg2, + ) -> None: + pass + + + def func2( + arg1, + arg2, + ): + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testTwoWordComparisonOperators(self): + unformatted_code = textwrap.dedent("""\ + _ = (klsdfjdklsfjksdlfjdklsfjdslkfjsdkl is not ksldfjsdklfjdklsfjdklsfjdklsfjdsklfjdklsfj) + _ = (klsdfjdklsfjksdlfjdklsfjdslkfjsdkl not in {ksldfjsdklfjdklsfjdklsfjdklsfjdsklfjdklsfj}) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + _ = (klsdfjdklsfjksdlfjdklsfjdslkfjsdkl + is not ksldfjsdklfjdklsfjdklsfjdklsfjdsklfjdklsfj) + _ = (klsdfjdklsfjksdlfjdklsfjdslkfjsdkl + not in {ksldfjsdklfjdklsfjdklsfjdklsfjdsklfjdklsfj}) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testStableInlinedDictionaryFormatting(self): + unformatted_code = textwrap.dedent("""\ + def _(): + url = "http://{0}/axis-cgi/admin/param.cgi?{1}".format( + value, urllib.urlencode({'action': 'update', 'parameter': value})) + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + def _(): + url = "http://{0}/axis-cgi/admin/param.cgi?{1}".format( + value, urllib.urlencode({ + 'action': 'update', + 'parameter': value + })) + """) + + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + reformatted_code = reformatter.Reformat(llines) + self.assertCodeEqual(expected_formatted_code, reformatted_code) + + llines = yapf_test_helper.ParseAndUnwrap(reformatted_code) + reformatted_code = reformatter.Reformat(llines) + self.assertCodeEqual(expected_formatted_code, reformatted_code) + + +class TestsForSpacesInsideBrackets(yapf_test_helper.YAPFTest): + """Test the SPACE_INSIDE_BRACKETS style option.""" + unformatted_code = textwrap.dedent("""\ + foo() + foo(1) + foo(1,2) + foo((1,)) + foo((1, 2)) + foo((1, 2,)) + foo(bar['baz'][0]) + set1 = {1, 2, 3} + dict1 = {1: 1, foo: 2, 3: bar} + dict2 = { + 1: 1, + foo: 2, + 3: bar, + } + dict3[3][1][get_index(*args,**kwargs)] + dict4[3][1][get_index(**kwargs)] + x = dict5[4](foo(*args)) + a = list1[:] + b = list2[slice_start:] + c = list3[slice_start:slice_end] + d = list4[slice_start:slice_end:] + e = list5[slice_start:slice_end:slice_step] + # Print gets special handling + print(set2) + compound = ((10+3)/(5-2**(6+x))) + string_idx = "mystring"[3] + """) + + def testEnabled(self): + style.SetGlobalStyle( + style.CreateStyleFromConfig('{space_inside_brackets: True}')) + + expected_formatted_code = textwrap.dedent("""\ + foo() + foo( 1 ) + foo( 1, 2 ) + foo( ( 1, ) ) + foo( ( 1, 2 ) ) + foo( ( + 1, + 2, + ) ) + foo( bar[ 'baz' ][ 0 ] ) + set1 = { 1, 2, 3 } + dict1 = { 1: 1, foo: 2, 3: bar } + dict2 = { + 1: 1, + foo: 2, + 3: bar, + } + dict3[ 3 ][ 1 ][ get_index( *args, **kwargs ) ] + dict4[ 3 ][ 1 ][ get_index( **kwargs ) ] + x = dict5[ 4 ]( foo( *args ) ) + a = list1[ : ] + b = list2[ slice_start: ] + c = list3[ slice_start:slice_end ] + d = list4[ slice_start:slice_end: ] + e = list5[ slice_start:slice_end:slice_step ] + # Print gets special handling + print( set2 ) + compound = ( ( 10 + 3 ) / ( 5 - 2**( 6 + x ) ) ) + string_idx = "mystring"[ 3 ] + """) + + llines = yapf_test_helper.ParseAndUnwrap(self.unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testDefault(self): + style.SetGlobalStyle(style.CreatePEP8Style()) + + expected_formatted_code = textwrap.dedent("""\ + foo() + foo(1) + foo(1, 2) + foo((1, )) + foo((1, 2)) + foo(( + 1, + 2, + )) + foo(bar['baz'][0]) + set1 = {1, 2, 3} + dict1 = {1: 1, foo: 2, 3: bar} + dict2 = { + 1: 1, + foo: 2, + 3: bar, + } + dict3[3][1][get_index(*args, **kwargs)] + dict4[3][1][get_index(**kwargs)] + x = dict5[4](foo(*args)) + a = list1[:] + b = list2[slice_start:] + c = list3[slice_start:slice_end] + d = list4[slice_start:slice_end:] + e = list5[slice_start:slice_end:slice_step] + # Print gets special handling + print(set2) + compound = ((10 + 3) / (5 - 2**(6 + x))) + string_idx = "mystring"[3] + """) + + llines = yapf_test_helper.ParseAndUnwrap(self.unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + +class TestsForSpacesAroundSubscriptColon(yapf_test_helper.YAPFTest): + """Test the SPACES_AROUND_SUBSCRIPT_COLON style option.""" + unformatted_code = textwrap.dedent("""\ + a = list1[ : ] + b = list2[ slice_start: ] + c = list3[ slice_start:slice_end ] + d = list4[ slice_start:slice_end: ] + e = list5[ slice_start:slice_end:slice_step ] + a1 = list1[ : ] + b1 = list2[ 1: ] + c1 = list3[ 1:20 ] + d1 = list4[ 1:20: ] + e1 = list5[ 1:20:3 ] + """) + + def testEnabled(self): + style.SetGlobalStyle( + style.CreateStyleFromConfig('{spaces_around_subscript_colon: True}')) + expected_formatted_code = textwrap.dedent("""\ + a = list1[:] + b = list2[slice_start :] + c = list3[slice_start : slice_end] + d = list4[slice_start : slice_end :] + e = list5[slice_start : slice_end : slice_step] + a1 = list1[:] + b1 = list2[1 :] + c1 = list3[1 : 20] + d1 = list4[1 : 20 :] + e1 = list5[1 : 20 : 3] + """) + llines = yapf_test_helper.ParseAndUnwrap(self.unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testWithSpaceInsideBrackets(self): + style.SetGlobalStyle( + style.CreateStyleFromConfig('{spaces_around_subscript_colon: true, ' + 'space_inside_brackets: true,}')) + expected_formatted_code = textwrap.dedent("""\ + a = list1[ : ] + b = list2[ slice_start : ] + c = list3[ slice_start : slice_end ] + d = list4[ slice_start : slice_end : ] + e = list5[ slice_start : slice_end : slice_step ] + a1 = list1[ : ] + b1 = list2[ 1 : ] + c1 = list3[ 1 : 20 ] + d1 = list4[ 1 : 20 : ] + e1 = list5[ 1 : 20 : 3 ] + """) + llines = yapf_test_helper.ParseAndUnwrap(self.unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testDefault(self): + style.SetGlobalStyle(style.CreatePEP8Style()) + expected_formatted_code = textwrap.dedent("""\ + a = list1[:] + b = list2[slice_start:] + c = list3[slice_start:slice_end] + d = list4[slice_start:slice_end:] + e = list5[slice_start:slice_end:slice_step] + a1 = list1[:] + b1 = list2[1:] + c1 = list3[1:20] + d1 = list4[1:20:] + e1 = list5[1:20:3] + """) + llines = yapf_test_helper.ParseAndUnwrap(self.unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + +if __name__ == '__main__': + unittest.main() diff --git a/yapftests/reformatter_python3_test.py b/yapftests/reformatter_python3_test.py new file mode 100644 index 000000000..f5741b313 --- /dev/null +++ b/yapftests/reformatter_python3_test.py @@ -0,0 +1,569 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Python 3 tests for yapf.reformatter.""" + +import sys +import textwrap +import unittest + +from yapf.yapflib import reformatter +from yapf.yapflib import style + +from yapftests import yapf_test_helper + + +class TestsForPython3Code(yapf_test_helper.YAPFTest): + """Test a few constructs that are new Python 3 syntax.""" + + @classmethod + def setUpClass(cls): # pylint: disable=g-missing-super-call + style.SetGlobalStyle(style.CreatePEP8Style()) + + def testTypedNames(self): + unformatted_code = textwrap.dedent("""\ + def x(aaaaaaaaaaaaaaa:int,bbbbbbbbbbbbbbbb:str,ccccccccccccccc:dict,eeeeeeeeeeeeee:set={1, 2, 3})->bool: + pass + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + def x(aaaaaaaaaaaaaaa: int, + bbbbbbbbbbbbbbbb: str, + ccccccccccccccc: dict, + eeeeeeeeeeeeee: set = {1, 2, 3}) -> bool: + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testTypedNameWithLongNamedArg(self): + unformatted_code = textwrap.dedent("""\ + def func(arg=long_function_call_that_pushes_the_line_over_eighty_characters()) -> ReturnType: + pass + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + def func(arg=long_function_call_that_pushes_the_line_over_eighty_characters() + ) -> ReturnType: + pass + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testKeywordOnlyArgSpecifier(self): + unformatted_code = textwrap.dedent("""\ + def foo(a, *, kw): + return a+kw + """) + expected_formatted_code = textwrap.dedent("""\ + def foo(a, *, kw): + return a + kw + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testAnnotations(self): + unformatted_code = textwrap.dedent("""\ + def foo(a: list, b: "bar") -> dict: + return a+b + """) + expected_formatted_code = textwrap.dedent("""\ + def foo(a: list, b: "bar") -> dict: + return a + b + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testExecAsNonKeyword(self): + unformatted_code = 'methods.exec( sys.modules[name])\n' + expected_formatted_code = 'methods.exec(sys.modules[name])\n' + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testAsyncFunctions(self): + code = textwrap.dedent("""\ + import asyncio + import time + + + @print_args + async def slow_operation(): + await asyncio.sleep(1) + # print("Slow operation {} complete".format(n)) + + + async def main(): + start = time.time() + if (await get_html()): + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testNoSpacesAroundPowerOperator(self): + unformatted_code = textwrap.dedent("""\ + a**b + """) + expected_formatted_code = textwrap.dedent("""\ + a ** b + """) + + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{based_on_style: pep8, SPACES_AROUND_POWER_OPERATOR: True}')) + + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreatePEP8Style()) + + def testSpacesAroundDefaultOrNamedAssign(self): + unformatted_code = textwrap.dedent("""\ + f(a=5) + """) + expected_formatted_code = textwrap.dedent("""\ + f(a = 5) + """) + + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{based_on_style: pep8, ' + 'SPACES_AROUND_DEFAULT_OR_NAMED_ASSIGN: True}')) + + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreatePEP8Style()) + + def testTypeHint(self): + unformatted_code = textwrap.dedent("""\ + def foo(x: int=42): + pass + + + def foo2(x: 'int' =42): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + def foo(x: int = 42): + pass + + + def foo2(x: 'int' = 42): + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testMatrixMultiplication(self): + unformatted_code = textwrap.dedent("""\ + a=b@c + """) + expected_formatted_code = textwrap.dedent("""\ + a = b @ c + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testNoneKeyword(self): + code = textwrap.dedent("""\ + None.__ne__() + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testAsyncWithPrecedingComment(self): + unformatted_code = textwrap.dedent("""\ + import asyncio + + # Comment + async def bar(): + pass + + async def foo(): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + import asyncio + + + # Comment + async def bar(): + pass + + + async def foo(): + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testAsyncFunctionsNested(self): + code = textwrap.dedent("""\ + async def outer(): + + async def inner(): + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testKeepTypesIntact(self): + unformatted_code = textwrap.dedent("""\ + def _ReduceAbstractContainers( + self, *args: Optional[automation_converter.PyiCollectionAbc]) -> List[ + automation_converter.PyiCollectionAbc]: + pass + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + def _ReduceAbstractContainers( + self, *args: Optional[automation_converter.PyiCollectionAbc] + ) -> List[automation_converter.PyiCollectionAbc]: + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testContinuationIndentWithAsync(self): + unformatted_code = textwrap.dedent("""\ + async def start_websocket(): + async with session.ws_connect( + r"ws://a_really_long_long_long_long_long_long_url") as ws: + pass + """) + expected_formatted_code = textwrap.dedent("""\ + async def start_websocket(): + async with session.ws_connect( + r"ws://a_really_long_long_long_long_long_long_url") as ws: + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testSplittingArguments(self): + unformatted_code = textwrap.dedent("""\ + async def open_file(file, mode='r', buffering=-1, encoding=None, errors=None, newline=None, closefd=True, opener=None): + pass + + async def run_sync_in_worker_thread(sync_fn, *args, cancellable=False, limiter=None): + pass + + def open_file(file, mode='r', buffering=-1, encoding=None, errors=None, newline=None, closefd=True, opener=None): + pass + + def run_sync_in_worker_thread(sync_fn, *args, cancellable=False, limiter=None): + pass + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + async def open_file( + file, + mode='r', + buffering=-1, + encoding=None, + errors=None, + newline=None, + closefd=True, + opener=None + ): + pass + + + async def run_sync_in_worker_thread( + sync_fn, *args, cancellable=False, limiter=None + ): + pass + + + def open_file( + file, + mode='r', + buffering=-1, + encoding=None, + errors=None, + newline=None, + closefd=True, + opener=None + ): + pass + + + def run_sync_in_worker_thread(sync_fn, *args, cancellable=False, limiter=None): + pass + """) # noqa + + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{based_on_style: pep8, ' + 'dedent_closing_brackets: true, ' + 'coalesce_brackets: false, ' + 'space_between_ending_comma_and_closing_bracket: false, ' + 'split_arguments_when_comma_terminated: true, ' + 'split_before_first_argument: true}')) + + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreatePEP8Style()) + + def testDictUnpacking(self): + unformatted_code = textwrap.dedent("""\ + class Foo: + def foo(self): + foofoofoofoofoofoofoofoo('foofoofoofoofoo', { + + 'foo': 'foo', + + **foofoofoo + }) + """) + expected_formatted_code = textwrap.dedent("""\ + class Foo: + + def foo(self): + foofoofoofoofoofoofoofoo('foofoofoofoofoo', { + 'foo': 'foo', + **foofoofoo + }) + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testMultilineFormatString(self): + # https://github.com/google/yapf/issues/513 + code = textwrap.dedent("""\ + # yapf: disable + (f''' + ''') + # yapf: enable + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testEllipses(self): + # https://github.com/google/yapf/issues/533 + code = textwrap.dedent("""\ + def dirichlet(x12345678901234567890123456789012345678901234567890=...) -> None: + return + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testFunctionTypedReturnNextLine(self): + code = textwrap.dedent("""\ + def _GenerateStatsEntries( + process_id: Text, + timestamp: Optional[ffffffff.FFFFFFFFFFF] = None + ) -> Sequence[ssssssssssss.SSSSSSSSSSSSSSS]: + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testFunctionTypedReturnSameLine(self): + code = textwrap.dedent("""\ + def rrrrrrrrrrrrrrrrrrrrrr( + ccccccccccccccccccccccc: Tuple[Text, Text]) -> List[Tuple[Text, Text]]: + pass + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testAsyncForElseNotIndentedInsideBody(self): + code = textwrap.dedent("""\ + async def fn(): + async for message in websocket: + for i in range(10): + pass + else: + pass + else: + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testForElseInAsyncNotMixedWithAsyncFor(self): + code = textwrap.dedent("""\ + async def fn(): + for i in range(10): + pass + else: + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testParameterListIndentationConflicts(self): + unformatted_code = textwrap.dedent("""\ + def raw_message( # pylint: disable=too-many-arguments + self, text, user_id=1000, chat_type='private', forward_date=None, forward_from=None): + pass + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + def raw_message( # pylint: disable=too-many-arguments + self, + text, + user_id=1000, + chat_type='private', + forward_date=None, + forward_from=None): + pass + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testTypeHintedYieldExpression(self): + # https://github.com/google/yapf/issues/1092 + code = textwrap.dedent("""\ + def my_coroutine(): + x: int = yield + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testSyntaxMatch(self): + # https://github.com/google/yapf/issues/1045 + # https://github.com/google/yapf/issues/1085 + unformatted_code = textwrap.dedent("""\ + a=3 + b=0 + match a : + case 0 : + b=1 + case _ : + b=2 + """) + expected_formatted_code = textwrap.dedent("""\ + a = 3 + b = 0 + match a: + case 0: + b = 1 + case _: + b = 2 + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testParenthsizedContextManager(self): + # https://github.com/google/yapf/issues/1064 + unformatted_code = textwrap.dedent("""\ + def test_copy_dimension(self): + with (Dataset() as target_ds, + Dataset() as source_ds): + do_something + """) + expected_formatted_code = textwrap.dedent("""\ + def test_copy_dimension(self): + with (Dataset() as target_ds, Dataset() as source_ds): + do_something + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testUnpackedTuple(self): + # https://github.com/google/yapf/issues/830 + # https://github.com/google/yapf/issues/1060 + unformatted_code = textwrap.dedent("""\ + def a(): + t = (2,3) + for i in range(5): + yield i,*t + """) + expected_formatted_code = textwrap.dedent("""\ + def a(): + t = (2, 3) + for i in range(5): + yield i, *t + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testTypedTuple(self): + # https://github.com/google/yapf/issues/412 + # https://github.com/google/yapf/issues/1058 + code = textwrap.dedent("""\ + t: tuple = 1, 2 + args = tuple(x for x in [2], ) + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + def testWalrusOperator(self): + # https://github.com/google/yapf/issues/894 + unformatted_code = textwrap.dedent("""\ + import os + a=[1,2,3,4] + if (n:=len(a))>2: + print() + """) + expected_formatted_code = textwrap.dedent("""\ + import os + + a = [1, 2, 3, 4] + if (n := len(a)) > 2: + print() + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testCondAssign(self): + # https://github.com/google/yapf/issues/856 + unformatted_code = textwrap.dedent("""\ + def json(self) -> JSONTask: + result: JSONTask = { + "id": self.id, + "text": self.text, + "status": self.status, + "last_mod": self.last_mod_time + } + for i in "parent_id", "deadline", "reminder": + if x := getattr(self , i): + result[i] = x # type: ignore + return result + """) + expected_formatted_code = textwrap.dedent("""\ + def json(self) -> JSONTask: + result: JSONTask = { + "id": self.id, + "text": self.text, + "status": self.status, + "last_mod": self.last_mod_time + } + for i in "parent_id", "deadline", "reminder": + if x := getattr(self, i): + result[i] = x # type: ignore + return result + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testCopyDictionary(self): + # https://github.com/google/yapf/issues/233 + # https://github.com/google/yapf/issues/402 + code = textwrap.dedent("""\ + a_dict = {'key': 'value'} + a_dict_copy = {**a_dict} + print('a_dict:', a_dict) + print('a_dict_copy:', a_dict_copy) + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(llines)) + + +if __name__ == '__main__': + unittest.main() diff --git a/yapftests/reformatter_style_config_test.py b/yapftests/reformatter_style_config_test.py new file mode 100644 index 000000000..6458a0afd --- /dev/null +++ b/yapftests/reformatter_style_config_test.py @@ -0,0 +1,198 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Style config tests for yapf.reformatter.""" + +import textwrap +import unittest + +from yapf.yapflib import reformatter +from yapf.yapflib import style + +from yapftests import yapf_test_helper + + +class TestsForStyleConfig(yapf_test_helper.YAPFTest): + + def setUp(self): + self.current_style = style.DEFAULT_STYLE + + def testSetGlobalStyle(self): + try: + style.SetGlobalStyle(style.CreateYapfStyle()) + unformatted_code = textwrap.dedent("""\ + for i in range(5): + print('bar') + """) + expected_formatted_code = textwrap.dedent("""\ + for i in range(5): + print('bar') + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreatePEP8Style()) + style.DEFAULT_STYLE = self.current_style + + unformatted_code = textwrap.dedent("""\ + for i in range(5): + print('bar') + """) + expected_formatted_code = textwrap.dedent("""\ + for i in range(5): + print('bar') + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + + def testOperatorNoSpaceStyle(self): + try: + sympy_style = style.CreatePEP8Style() + sympy_style['NO_SPACES_AROUND_SELECTED_BINARY_OPERATORS'] = \ + style._StringSetConverter('*,/') + style.SetGlobalStyle(sympy_style) + unformatted_code = textwrap.dedent("""\ + a = 1+2 * 3 - 4 / 5 + b = '0' * 1 + """) + expected_formatted_code = textwrap.dedent("""\ + a = 1 + 2*3 - 4/5 + b = '0'*1 + """) + + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreatePEP8Style()) + style.DEFAULT_STYLE = self.current_style + + def testOperatorPrecedenceStyle(self): + try: + pep8_with_precedence = style.CreatePEP8Style() + pep8_with_precedence['ARITHMETIC_PRECEDENCE_INDICATION'] = True + style.SetGlobalStyle(pep8_with_precedence) + unformatted_code = textwrap.dedent("""\ + 1+2 + (1 + 2) * (3 - (4 / 5)) + a = 1 * 2 + 3 / 4 + b = 1 / 2 - 3 * 4 + c = (1 + 2) * (3 - 4) + d = (1 - 2) / (3 + 4) + e = 1 * 2 - 3 + f = 1 + 2 + 3 + 4 + g = 1 * 2 * 3 * 4 + h = 1 + 2 - 3 + 4 + i = 1 * 2 / 3 * 4 + j = (1 * 2 - 3) + 4 + k = (1 * 2 * 3) + (4 * 5 * 6 * 7 * 8) + """) + expected_formatted_code = textwrap.dedent("""\ + 1 + 2 + (1+2) * (3 - (4/5)) + a = 1*2 + 3/4 + b = 1/2 - 3*4 + c = (1+2) * (3-4) + d = (1-2) / (3+4) + e = 1*2 - 3 + f = 1 + 2 + 3 + 4 + g = 1 * 2 * 3 * 4 + h = 1 + 2 - 3 + 4 + i = 1 * 2 / 3 * 4 + j = (1*2 - 3) + 4 + k = (1*2*3) + (4*5*6*7*8) + """) + + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreatePEP8Style()) + style.DEFAULT_STYLE = self.current_style + + def testNoSplitBeforeFirstArgumentStyle1(self): + try: + pep8_no_split_before_first = style.CreatePEP8Style() + pep8_no_split_before_first['SPLIT_BEFORE_FIRST_ARGUMENT'] = False + pep8_no_split_before_first['SPLIT_BEFORE_NAMED_ASSIGNS'] = False + style.SetGlobalStyle(pep8_no_split_before_first) + formatted_code = textwrap.dedent("""\ + # Example from in-code MustSplit comments + foo = outer_function_call(fitting_inner_function_call(inner_arg1, inner_arg2), + outer_arg1, outer_arg2) + + foo = outer_function_call( + not_fitting_inner_function_call(inner_arg1, inner_arg2), outer_arg1, + outer_arg2) + + # Examples Issue#424 + a_super_long_version_of_print(argument1, argument2, argument3, argument4, + argument5, argument6, argument7) + + CREDS_FILE = os.path.join(os.path.expanduser('~'), + 'apis/super-secret-admin-creds.json') + + # Examples Issue#556 + i_take_a_lot_of_params(arg1, param1=very_long_expression1(), + param2=very_long_expression2(), + param3=very_long_expression3(), + param4=very_long_expression4()) + + # Examples Issue#590 + plt.plot(numpy.linspace(0, 1, 10), numpy.linspace(0, 1, 10), marker="x", + color="r") + + plt.plot(veryverylongvariablename, veryverylongvariablename, marker="x", + color="r") + """) # noqa + llines = yapf_test_helper.ParseAndUnwrap(formatted_code) + self.assertCodeEqual(formatted_code, reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreatePEP8Style()) + style.DEFAULT_STYLE = self.current_style + + def testNoSplitBeforeFirstArgumentStyle2(self): + try: + pep8_no_split_before_first = style.CreatePEP8Style() + pep8_no_split_before_first['SPLIT_BEFORE_FIRST_ARGUMENT'] = False + pep8_no_split_before_first['SPLIT_BEFORE_NAMED_ASSIGNS'] = True + style.SetGlobalStyle(pep8_no_split_before_first) + formatted_code = textwrap.dedent("""\ + # Examples Issue#556 + i_take_a_lot_of_params(arg1, + param1=very_long_expression1(), + param2=very_long_expression2(), + param3=very_long_expression3(), + param4=very_long_expression4()) + + # Examples Issue#590 + plt.plot(numpy.linspace(0, 1, 10), + numpy.linspace(0, 1, 10), + marker="x", + color="r") + + plt.plot(veryverylongvariablename, + veryverylongvariablename, + marker="x", + color="r") + """) + llines = yapf_test_helper.ParseAndUnwrap(formatted_code) + self.assertCodeEqual(formatted_code, reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreatePEP8Style()) + style.DEFAULT_STYLE = self.current_style + + +if __name__ == '__main__': + unittest.main() diff --git a/yapftests/reformatter_test.py b/yapftests/reformatter_test.py deleted file mode 100644 index cb2e6d5f3..000000000 --- a/yapftests/reformatter_test.py +++ /dev/null @@ -1,3657 +0,0 @@ -# Copyright 2015-2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Tests for yapf.reformatter.""" - -import difflib -import sys -import textwrap -import unittest - -from yapf.yapflib import blank_line_calculator -from yapf.yapflib import comment_splicer -from yapf.yapflib import continuation_splicer -from yapf.yapflib import py3compat -from yapf.yapflib import pytree_unwrapper -from yapf.yapflib import pytree_utils -from yapf.yapflib import pytree_visitor -from yapf.yapflib import reformatter -from yapf.yapflib import split_penalty -from yapf.yapflib import style -from yapf.yapflib import subtype_assigner -from yapf.yapflib import verifier - - -class ReformatterTest(unittest.TestCase): - - def assertCodeEqual(self, expected_code, code): - if code != expected_code: - msg = ['Code format mismatch:', 'Expected:'] - linelen = style.Get('COLUMN_LIMIT') - for l in expected_code.splitlines(): - if len(l) > linelen: - msg.append('!> %s' % l) - else: - msg.append(' > %s' % l) - msg.append('Actual:') - for l in code.splitlines(): - if len(l) > linelen: - msg.append('!> %s' % l) - else: - msg.append(' > %s' % l) - msg.append('Diff:') - msg.extend( - difflib.unified_diff( - code.splitlines(), - expected_code.splitlines(), - fromfile='actual', - tofile='expected', - lineterm='')) - self.fail('\n'.join(msg)) - - -class BasicReformatterTest(ReformatterTest): - - @classmethod - def setUpClass(cls): - style.SetGlobalStyle(style.CreateChromiumStyle()) - - def testSimple(self): - unformatted_code = textwrap.dedent("""\ - if a+b: - pass - """) - expected_formatted_code = textwrap.dedent("""\ - if a + b: - pass - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testSimpleFunctions(self): - unformatted_code = textwrap.dedent("""\ - def g(): - pass - - def f(): - pass - """) - expected_formatted_code = textwrap.dedent("""\ - def g(): - pass - - - def f(): - pass - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testSimpleFunctionsWithTrailingComments(self): - unformatted_code = textwrap.dedent("""\ - def g(): # Trailing comment - if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and - xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): - pass - - def f( # Intermediate comment - ): - if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and - xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): - pass - """) - expected_formatted_code = textwrap.dedent("""\ - def g(): # Trailing comment - if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and - xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): - pass - - - def f( # Intermediate comment - ): - if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and - xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): - pass - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testBlankLinesAtEndOfFile(self): - unformatted_code = textwrap.dedent("""\ - def foobar(): # foo - pass - - - - """) - expected_formatted_code = textwrap.dedent("""\ - def foobar(): # foo - pass - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - unformatted_code = textwrap.dedent("""\ - x = { 'a':37,'b':42, - - 'c':927} - - """) - expected_formatted_code = textwrap.dedent("""\ - x = {'a': 37, 'b': 42, 'c': 927} - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testMultipleUgliness(self): - unformatted_code = textwrap.dedent("""\ - x = { 'a':37,'b':42, - - 'c':927} - - y = 'hello ''world' - z = 'hello '+'world' - a = 'hello {}'.format('world') - class foo ( object ): - def f (self ): - return 37*-+2 - def g(self, x,y=42): - return y - def f ( a ) : - return 37+-+a[42-x : y**3] - """) - expected_formatted_code = textwrap.dedent("""\ - x = {'a': 37, 'b': 42, 'c': 927} - - y = 'hello ' 'world' - z = 'hello ' + 'world' - a = 'hello {}'.format('world') - - - class foo(object): - - def f(self): - return 37 * -+2 - - def g(self, x, y=42): - return y - - - def f(a): - return 37 + -+a[42 - x:y**3] - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testComments(self): - unformatted_code = textwrap.dedent("""\ - class Foo(object): - pass - # End class Foo - - # Attached comment - class Bar(object): - pass - - global_assignment = 42 - - # Comment attached to class with decorator. - # Comment attached to class with decorator. - @noop - @noop - class Baz(object): - pass - - # Intermediate comment - - class Qux(object): - pass - """) - expected_formatted_code = textwrap.dedent("""\ - class Foo(object): - pass - # End class Foo - - - # Attached comment - class Bar(object): - pass - - - global_assignment = 42 - - - # Comment attached to class with decorator. - # Comment attached to class with decorator. - @noop - @noop - class Baz(object): - pass - - # Intermediate comment - - - class Qux(object): - pass - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testSingleComment(self): - code = textwrap.dedent("""\ - # Thing 1 - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testCommentsInDataLiteral(self): - code = textwrap.dedent("""\ - def f(): - return collections.OrderedDict({ - # First comment. - 'fnord': 37, - - # Second comment. - # Continuation of second comment. - 'bork': 42, - - # Ending comment. - }) - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testEndingWhitespaceAfterSimpleStatement(self): - code = textwrap.dedent("""\ - import foo as bar - # Thing 1 - # Thing 2 - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testDocstrings(self): - unformatted_code = textwrap.dedent('''\ - u"""Module-level docstring.""" - import os - class Foo(object): - - """Class-level docstring.""" - # A comment for qux. - def qux(self): - - - """Function-level docstring. - - A multiline function docstring. - """ - print('hello {}'.format('world')) - return 42 - ''') - expected_formatted_code = textwrap.dedent('''\ - u"""Module-level docstring.""" - import os - - - class Foo(object): - """Class-level docstring.""" - - # A comment for qux. - def qux(self): - """Function-level docstring. - - A multiline function docstring. - """ - print('hello {}'.format('world')) - return 42 - ''') - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testDocstringAndMultilineComment(self): - unformatted_code = textwrap.dedent('''\ - """Hello world""" - # A multiline - # comment - class bar(object): - """class docstring""" - # class multiline - # comment - def foo(self): - """Another docstring.""" - # Another multiline - # comment - pass - ''') - expected_formatted_code = textwrap.dedent('''\ - """Hello world""" - - - # A multiline - # comment - class bar(object): - """class docstring""" - - # class multiline - # comment - def foo(self): - """Another docstring.""" - # Another multiline - # comment - pass - ''') - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testMultilineDocstringAndMultilineComment(self): - unformatted_code = textwrap.dedent('''\ - """Hello world - - RIP Dennis Richie. - """ - # A multiline - # comment - class bar(object): - """class docstring - - A classy class. - """ - # class multiline - # comment - def foo(self): - """Another docstring. - - A functional function. - """ - # Another multiline - # comment - pass - ''') - expected_formatted_code = textwrap.dedent('''\ - """Hello world - - RIP Dennis Richie. - """ - - - # A multiline - # comment - class bar(object): - """class docstring - - A classy class. - """ - - # class multiline - # comment - def foo(self): - """Another docstring. - - A functional function. - """ - # Another multiline - # comment - pass - ''') - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testTupleCommaBeforeLastParen(self): - unformatted_code = textwrap.dedent("""\ - a = ( 1, ) - """) - expected_formatted_code = textwrap.dedent("""\ - a = (1,) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testNoBreakOutsideOfBracket(self): - # FIXME(morbo): How this is formatted is not correct. But it's syntactically - # correct. - unformatted_code = textwrap.dedent("""\ - def f(): - assert port >= minimum, \ -'Unexpected port %d when minimum was %d.' % (port, minimum) - """) - expected_formatted_code = textwrap.dedent("""\ - def f(): - assert port >= minimum, 'Unexpected port %d when minimum was %d.' % (port, - minimum) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testBlankLinesBeforeDecorators(self): - unformatted_code = textwrap.dedent("""\ - @foo() - class A(object): - @bar() - @baz() - def x(self): - pass - """) - expected_formatted_code = textwrap.dedent("""\ - @foo() - class A(object): - - @bar() - @baz() - def x(self): - pass - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testCommentBetweenDecorators(self): - unformatted_code = textwrap.dedent("""\ - @foo() - # frob - @bar - def x (self): - pass - """) - expected_formatted_code = textwrap.dedent("""\ - @foo() - # frob - @bar - def x(self): - pass - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testListComprehension(self): - unformatted_code = textwrap.dedent("""\ - def given(y): - [k for k in () - if k in y] - """) - expected_formatted_code = textwrap.dedent("""\ - def given(y): - [k for k in () if k in y] - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testOpeningAndClosingBrackets(self): - unformatted_code = textwrap.dedent("""\ - foo( ( 1, 2, 3, ) ) - """) - expected_formatted_code = textwrap.dedent("""\ - foo((1, - 2, - 3,)) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testSingleLineFunctions(self): - unformatted_code = textwrap.dedent("""\ - def foo(): return 42 - """) - expected_formatted_code = textwrap.dedent("""\ - def foo(): - return 42 - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testNoQueueSeletionInMiddleOfLine(self): - # If the queue isn't properly consttructed, then a token in the middle of - # the line may be selected as the one with least penalty. The tokens after - # that one are then splatted at the end of the line with no formatting. - # FIXME(morbo): The formatting here isn't ideal. - unformatted_code = textwrap.dedent("""\ - find_symbol(node.type) + "< " + " ".join(find_pattern(n) for n in \ -node.child) + " >" - """) - expected_formatted_code = textwrap.dedent("""\ - find_symbol(node.type) + "< " + " ".join(find_pattern(n) - for n in node.child) + " >" - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testNoSpacesBetweenSubscriptsAndCalls(self): - unformatted_code = textwrap.dedent("""\ - aaaaaaaaaa = bbbbbbbb.ccccccccc() [42] (a, 2) - """) - expected_formatted_code = textwrap.dedent("""\ - aaaaaaaaaa = bbbbbbbb.ccccccccc()[42](a, 2) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testNoSpacesBetweenOpeningBracketAndStartingOperator(self): - # Unary operator. - unformatted_code = textwrap.dedent("""\ - aaaaaaaaaa = bbbbbbbb.ccccccccc[ -1 ]( -42 ) - """) - expected_formatted_code = textwrap.dedent("""\ - aaaaaaaaaa = bbbbbbbb.ccccccccc[-1](-42) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - # Varargs and kwargs. - unformatted_code = textwrap.dedent("""\ - aaaaaaaaaa = bbbbbbbb.ccccccccc( *varargs ) - aaaaaaaaaa = bbbbbbbb.ccccccccc( **kwargs ) - """) - expected_formatted_code = textwrap.dedent("""\ - aaaaaaaaaa = bbbbbbbb.ccccccccc(*varargs) - aaaaaaaaaa = bbbbbbbb.ccccccccc(**kwargs) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testMultilineCommentReformatted(self): - unformatted_code = textwrap.dedent("""\ - if True: - # This is a multiline - # comment. - pass - """) - expected_formatted_code = textwrap.dedent("""\ - if True: - # This is a multiline - # comment. - pass - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testDictionaryMakerFormatting(self): - unformatted_code = textwrap.dedent("""\ - _PYTHON_STATEMENTS = frozenset({ - lambda x, y: 'simple_stmt': 'small_stmt', 'expr_stmt': 'print_stmt', 'del_stmt': - 'pass_stmt', lambda: 'break_stmt': 'continue_stmt', 'return_stmt': 'raise_stmt', - 'yield_stmt': 'import_stmt', lambda: 'global_stmt': 'exec_stmt', 'assert_stmt': - 'if_stmt', 'while_stmt': 'for_stmt', - }) - """) - expected_formatted_code = textwrap.dedent("""\ - _PYTHON_STATEMENTS = frozenset({ - lambda x, y: 'simple_stmt': 'small_stmt', - 'expr_stmt': 'print_stmt', - 'del_stmt': 'pass_stmt', - lambda: 'break_stmt': 'continue_stmt', - 'return_stmt': 'raise_stmt', - 'yield_stmt': 'import_stmt', - lambda: 'global_stmt': 'exec_stmt', - 'assert_stmt': 'if_stmt', - 'while_stmt': 'for_stmt', - }) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testSimpleMultilineCode(self): - unformatted_code = textwrap.dedent("""\ - if True: - aaaaaaaaaaaaaa.bbbbbbbbbbbbbb.ccccccc(zzzzzzzzzzzz, \ -xxxxxxxxxxx, yyyyyyyyyyyy, vvvvvvvvv) - aaaaaaaaaaaaaa.bbbbbbbbbbbbbb.ccccccc(zzzzzzzzzzzz, \ -xxxxxxxxxxx, yyyyyyyyyyyy, vvvvvvvvv) - """) - expected_formatted_code = textwrap.dedent("""\ - if True: - aaaaaaaaaaaaaa.bbbbbbbbbbbbbb.ccccccc(zzzzzzzzzzzz, xxxxxxxxxxx, yyyyyyyyyyyy, - vvvvvvvvv) - aaaaaaaaaaaaaa.bbbbbbbbbbbbbb.ccccccc(zzzzzzzzzzzz, xxxxxxxxxxx, yyyyyyyyyyyy, - vvvvvvvvv) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testMultilineComment(self): - code = textwrap.dedent("""\ - if Foo: - # Hello world - # Yo man. - # Yo man. - # Yo man. - # Yo man. - a = 42 - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testMultilineString(self): - code = textwrap.dedent("""\ - code = textwrap.dedent('''\ - if Foo: - # Hello world - # Yo man. - # Yo man. - # Yo man. - # Yo man. - a = 42 - ''') - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - unformatted_code = textwrap.dedent('''\ - def f(): - email_text += """This is a really long docstring that goes over the column limit and is multi-line.

- Czar: """+despot["Nicholas"]+"""
- Minion: """+serf["Dmitri"]+"""
- Residence: """+palace["Winter"]+"""
- - """ - ''') - expected_formatted_code = textwrap.dedent('''\ - def f(): - email_text += """This is a really long docstring that goes over the column limit and is multi-line.

- Czar: """ + despot["Nicholas"] + """
- Minion: """ + serf["Dmitri"] + """
- Residence: """ + palace["Winter"] + """
- - """ - ''') - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testSimpleMultilineWithComments(self): - code = textwrap.dedent("""\ - if ( # This is the first comment - a and # This is the second comment - # This is the third comment - b): # A trailing comment - # Whoa! A normal comment!! - pass # Another trailing comment - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testMatchingParenSplittingMatching(self): - unformatted_code = textwrap.dedent("""\ - def f(): - raise RuntimeError('unable to find insertion point for target node', - (target,)) - """) - expected_formatted_code = textwrap.dedent("""\ - def f(): - raise RuntimeError('unable to find insertion point for target node', - (target,)) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testContinuationIndent(self): - unformatted_code = textwrap.dedent('''\ - class F: - def _ProcessArgLists(self, node): - """Common method for processing argument lists.""" - for child in node.children: - if isinstance(child, pytree.Leaf): - self._SetTokenSubtype( - child, subtype=_ARGLIST_TOKEN_TO_SUBTYPE.get( - child.value, format_token.Subtype.NONE)) - ''') - expected_formatted_code = textwrap.dedent('''\ - class F: - - def _ProcessArgLists(self, node): - """Common method for processing argument lists.""" - for child in node.children: - if isinstance(child, pytree.Leaf): - self._SetTokenSubtype( - child, - subtype=_ARGLIST_TOKEN_TO_SUBTYPE.get(child.value, - format_token.Subtype.NONE)) - ''') - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testTrailingCommaAndBracket(self): - unformatted_code = textwrap.dedent('''\ - a = { 42, } - b = ( 42, ) - c = [ 42, ] - ''') - expected_formatted_code = textwrap.dedent('''\ - a = {42,} - b = (42,) - c = [42,] - ''') - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testI18n(self): - code = textwrap.dedent("""\ - N_('Some years ago - never mind how long precisely - having little or no money in my purse, and nothing particular to interest me on shore, I thought I would sail about a little and see the watery part of the world.') # A comment is here. - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - code = textwrap.dedent("""\ - foo('Fake function call') #. Some years ago - never mind how long precisely - having little or no money in my purse, and nothing particular to interest me on shore, I thought I would sail about a little and see the watery part of the world. - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testI18nCommentsInDataLiteral(self): - code = textwrap.dedent("""\ - def f(): - return collections.OrderedDict({ - #. First i18n comment. - 'bork': 'foo', - - #. Second i18n comment. - 'snork': 'bar#.*=\\\\0', - }) - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testClosingBracketIndent(self): - code = textwrap.dedent('''\ - def f(): - - def g(): - while ( - xxxxxxxxxxxxxxxxxxxxx(yyyyyyyyyyyyy[zzzzz]) == 'aaaaaaaaaaa' and - xxxxxxxxxxxxxxxxxxxxx(yyyyyyyyyyyyy[zzzzz].aaaaaaaa[0]) == 'bbbbbbb'): - pass - ''') - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testClosingBracketsInlinedInCall(self): - unformatted_code = textwrap.dedent("""\ - class Foo(object): - - def bar(self): - self.aaaaaaaa = xxxxxxxxxxxxxxxxxxx.yyyyyyyyyyyyy( - self.cccccc.ddddddddd.eeeeeeee, - options={ - "forkforkfork": 1, - "borkborkbork": 2, - "corkcorkcork": 3, - "horkhorkhork": 4, - "porkporkpork": 5, - }) - """) - expected_formatted_code = textwrap.dedent("""\ - class Foo(object): - - def bar(self): - self.aaaaaaaa = xxxxxxxxxxxxxxxxxxx.yyyyyyyyyyyyy( - self.cccccc.ddddddddd.eeeeeeee, - options={ - "forkforkfork": 1, - "borkborkbork": 2, - "corkcorkcork": 3, - "horkhorkhork": 4, - "porkporkpork": 5, - }) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testLineWrapInForExpression(self): - code = textwrap.dedent("""\ - class A: - - def x(self, node, name, n=1): - for i, child in enumerate( - itertools.ifilter(lambda c: pytree_utils.NodeName(c) == name, - node.pre_order())): - pass - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testFunctionCallContinuationLine(self): - code = textwrap.dedent("""\ - class foo: - - def bar(self, node, name, n=1): - if True: - if True: - return [(aaaaaaaaaa, bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb( - cccc, ddddddddddddddddddddddddddddddddddddd))] - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testI18nNonFormatting(self): - code = textwrap.dedent("""\ - class F(object): - - def __init__(self, fieldname, - #. Error message indicating an invalid e-mail address. - message=N_('Please check your email address.'), **kwargs): - pass - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testNoSpaceBetweenUnaryOpAndOpeningParen(self): - code = textwrap.dedent("""\ - if ~(a or b): - pass - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testCommentBeforeFuncDef(self): - code = textwrap.dedent("""\ - class Foo(object): - - a = 42 - - # This is a comment. - def __init__(self, - xxxxxxx, - yyyyy=0, - zzzzzzz=None, - aaaaaaaaaaaaaaaaaa=False, - bbbbbbbbbbbbbbb=False): - pass - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testExcessLineCountWithDefaultKeywords(self): - unformatted_code = textwrap.dedent("""\ - class Fnord(object): - def Moo(self): - aaaaaaaaaaaaaaaa = self._bbbbbbbbbbbbbbbbbbbbbbb( - ccccccccccccc=ccccccccccccc, ddddddd=ddddddd, eeee=eeee, - fffff=fffff, ggggggg=ggggggg, hhhhhhhhhhhhh=hhhhhhhhhhhhh, - iiiiiii=iiiiiiiiiiiiii) - """) - expected_formatted_code = textwrap.dedent("""\ - class Fnord(object): - - def Moo(self): - aaaaaaaaaaaaaaaa = self._bbbbbbbbbbbbbbbbbbbbbbb( - ccccccccccccc=ccccccccccccc, - ddddddd=ddddddd, - eeee=eeee, - fffff=fffff, - ggggggg=ggggggg, - hhhhhhhhhhhhh=hhhhhhhhhhhhh, - iiiiiii=iiiiiiiiiiiiii) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testSpaceAfterNotOperator(self): - code = textwrap.dedent("""\ - if not (this and that): - pass - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testNoPenaltySplitting(self): - code = textwrap.dedent("""\ - def f(): - if True: - if True: - python_files.extend( - os.path.join(filename, f) for f in os.listdir(filename) - if IsPythonFile(os.path.join(filename, f))) - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testExpressionPenalties(self): - code = textwrap.dedent("""\ - def f(): - if ((left.value == '(' and right.value == ')') or - (left.value == '[' and right.value == ']') or - (left.value == '{' and right.value == '}')): - return False - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testLineDepthOfSingleLineStatement(self): - unformatted_code = textwrap.dedent("""\ - while True: continue - for x in range(3): continue - try: a = 42 - except: b = 42 - with open(a) as fd: a = fd.read() - """) - expected_formatted_code = textwrap.dedent("""\ - while True: - continue - for x in range(3): - continue - try: - a = 42 - except: - b = 42 - with open(a) as fd: - a = fd.read() - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testSplitListWithTerminatingComma(self): - unformatted_code = textwrap.dedent("""\ - FOO = ['bar', 'baz', 'mux', 'qux', 'quux', 'quuux', 'quuuux', - 'quuuuux', 'quuuuuux', 'quuuuuuux', lambda a, b: 37,] - """) - expected_formatted_code = textwrap.dedent("""\ - FOO = ['bar', - 'baz', - 'mux', - 'qux', - 'quux', - 'quuux', - 'quuuux', - 'quuuuux', - 'quuuuuux', - 'quuuuuuux', - lambda a, b: 37,] - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testSplitListWithInterspersedComments(self): - code = textwrap.dedent("""\ - FOO = ['bar', # bar - 'baz', # baz - 'mux', # mux - 'qux', # qux - 'quux', # quux - 'quuux', # quuux - 'quuuux', # quuuux - 'quuuuux', # quuuuux - 'quuuuuux', # quuuuuux - 'quuuuuuux', # quuuuuuux - lambda a, b: 37 # lambda - ] - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testRelativeImportStatements(self): - code = textwrap.dedent("""\ - from ... import bork - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testSingleLineList(self): - # A list on a single line should prefer to remain contiguous. - unformatted_code = textwrap.dedent("""\ - bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb = aaaaaaaaaaa( - ("...", "."), "..", - ".............................................." - ) - """) - expected_formatted_code = textwrap.dedent("""\ - bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb = aaaaaaaaaaa( - ("...", "."), "..", "..............................................") - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testBlankLinesBeforeFunctionsNotInColumnZero(self): - unformatted_code = textwrap.dedent("""\ - import signal - - - try: - signal.SIGALRM - # .................................................................. - # ............................................................... - - - def timeout(seconds=1): - pass - except: - pass - """) - expected_formatted_code = textwrap.dedent("""\ - import signal - - try: - signal.SIGALRM - - # .................................................................. - # ............................................................... - - - def timeout(seconds=1): - pass - except: - pass - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testNoKeywordArgumentBreakage(self): - code = textwrap.dedent("""\ - class A(object): - - def b(self): - if self.aaaaaaaaaaaaaaaaaaaa not in self.bbbbbbbbbb( - cccccccccccccccccccc=True): - pass - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testTrailerOnSingleLine(self): - code = textwrap.dedent("""\ - urlpatterns = patterns('', url(r'^$', 'homepage_view'), - url(r'^/login/$', 'login_view'), - url(r'^/login/$', 'logout_view'), - url(r'^/user/(?P\\w+)/$', 'profile_view')) - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testIfConditionalParens(self): - code = textwrap.dedent("""\ - class Foo: - - def bar(): - if True: - if (child.type == grammar_token.NAME and - child.value in substatement_names): - pass - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testContinuationMarkers(self): - code = textwrap.dedent("""\ - text = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec a diam lectus. "\\ - "Sed sit amet ipsum mauris. Maecenas congue ligula ac quam viverra nec consectetur "\\ - "ante hendrerit. Donec et mollis dolor. Praesent et diam eget libero egestas mattis "\\ - "sit amet vitae augue. Nam tincidunt congue enim, ut porta lorem lacinia consectetur. "\\ - "Donec ut libero sed arcu vehicula ultricies a non tortor. Lorem ipsum dolor sit amet" - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - code = textwrap.dedent("""\ - from __future__ import nested_scopes, generators, division, absolute_import, with_statement, \\ - print_function, unicode_literals - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - code = textwrap.dedent("""\ - if aaaaaaaaa == 42 and bbbbbbbbbbbbbb == 42 and \\ - cccccccc == 42: - pass - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testCommentsWithContinuationMarkers(self): - code = textwrap.dedent("""\ - def fn(arg): - v = fn2(key1=True, - #c1 - key2=arg)\\ - .fn3() - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testEmptyContainers(self): - code = textwrap.dedent("""\ - flags.DEFINE_list( - 'output_dirs', [], - 'Lorem ipsum dolor sit amet, consetetur adipiscing elit. Donec a diam lectus. ' - 'Sed sit amet ipsum mauris. Maecenas congue.') - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testSplitStringsIfSurroundedByParens(self): - unformatted_code = textwrap.dedent("""\ - a = foo.bar({'xxxxxxxxxxxxxxxxxxxxxxx' 'yyyyyyyyyyyyyyyyyyyyyyyyyy': baz[42]} + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' 'bbbbbbbbbbbbbbbbbbbbbbbbbb' 'cccccccccccccccccccccccccccccccc' 'ddddddddddddddddddddddddddddd') - """) - expected_formatted_code = textwrap.dedent("""\ - a = foo.bar({'xxxxxxxxxxxxxxxxxxxxxxx' - 'yyyyyyyyyyyyyyyyyyyyyyyyyy': baz[42]} + - 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' - 'bbbbbbbbbbbbbbbbbbbbbbbbbb' - 'cccccccccccccccccccccccccccccccc' - 'ddddddddddddddddddddddddddddd') - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - code = textwrap.dedent("""\ - a = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' \ -'bbbbbbbbbbbbbbbbbbbbbbbbbb' 'cccccccccccccccccccccccccccccccc' \ -'ddddddddddddddddddddddddddddd' - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testMultilineShebang(self): - code = textwrap.dedent("""\ - #!/bin/sh - if "true" : '''\' - then - - export FOO=123 - exec /usr/bin/env python "$0" "$@" - - exit 127 - fi - ''' - - import os - - assert os.environ['FOO'] == '123' - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testNoSplittingAroundTermOperators(self): - code = textwrap.dedent("""\ - a_very_long_function_call_yada_yada_etc_etc_etc(long_arg1, - long_arg2 / long_arg3) - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testNoSplittingWithinSubscriptList(self): - code = textwrap.dedent("""\ - somequitelongvariablename.somemember[(a, b)] = { - 'somelongkey': 1, - 'someotherlongkey': 2 - } - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testExcessCharacters(self): - code = textwrap.dedent("""\ - class foo: - - def bar(self): - self.write(s=['%s%s %s' % ('many of really', 'long strings', - '+ just makes up 81')]) - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testDictSetGenerator(self): - code = textwrap.dedent("""\ - foo = { - variable: 'hello world. How are you today?' - for variable in fnord if variable != 37 - } - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testUnaryOpInDictionaryValue(self): - code = textwrap.dedent("""\ - beta = "123" - - test = {'alpha': beta[-1]} - - print(beta[-1]) - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testUnaryNotOperator(self): - code = textwrap.dedent("""\ - if True: - if True: - if True: - if True: - remote_checksum = self.get_checksum(conn, tmp, dest, inject, - not directory_prepended, source) - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testRelaxArraySubscriptAffinity(self): - code = textwrap.dedent("""\ - class A(object): - - def f(self, aaaaaaaaa, bbbbbbbbbbbbb, row): - if True: - if True: - if True: - if True: - if row[4] is None or row[5] is None: - bbbbbbbbbbbbb['..............'] = row[5] if row[ - 5] is not None else 5 - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testFunctionCallInDict(self): - code = "a = {'a': b(c=d, **e)}\n" - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testFunctionCallInNestedDict(self): - code = "a = {'a': {'a': {'a': b(c=d, **e)}}}\n" - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testUnbreakableNot(self): - code = textwrap.dedent("""\ - def test(): - if not "Foooooooooooooooooooooooooooooo" or "Foooooooooooooooooooooooooooooo" == "Foooooooooooooooooooooooooooooo": - pass - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testSplitListWithComment(self): - code = textwrap.dedent("""\ - a = [ - 'a', - 'b', - 'c' # hello world - ] - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testOverColumnLimit(self): - unformatted_code = textwrap.dedent("""\ - class Test: - - def testSomething(self): - expected = { - ('aaaaaaaaaaaaa', 'bbbb'): 'ccccccccccccccccccccccccccccccccccccccccccc', - ('aaaaaaaaaaaaa', 'bbbb'): 'ccccccccccccccccccccccccccccccccccccccccccc', - ('aaaaaaaaaaaaa', 'bbbb'): 'ccccccccccccccccccccccccccccccccccccccccccc', - } - """) - expected_formatted_code = textwrap.dedent("""\ - class Test: - - def testSomething(self): - expected = { - ('aaaaaaaaaaaaa', - 'bbbb'): 'ccccccccccccccccccccccccccccccccccccccccccc', - ('aaaaaaaaaaaaa', - 'bbbb'): 'ccccccccccccccccccccccccccccccccccccccccccc', - ('aaaaaaaaaaaaa', - 'bbbb'): 'ccccccccccccccccccccccccccccccccccccccccccc', - } - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testEndingComment(self): - code = textwrap.dedent("""\ - a = f(a="something", - b="something requiring comment which is quite long", # comment about b (pushes line over 79) - c="something else, about which comment doesn't make sense") - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testContinuationSpaceRetention(self): - code = textwrap.dedent("""\ - def fn(): - return module \\ - .method(Object(data, - fn2(arg) - )) - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testIfExpressionWithFunctionCall(self): - code = textwrap.dedent("""\ - if x or z.y(a, - c, - aaaaaaaaaaaaaaaaaaaaa=aaaaaaaaaaaaaaaaaa, - bbbbbbbbbbbbbbbbbbbbb=bbbbbbbbbbbbbbbbbb): - pass - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testUnformattedAfterMultilineString(self): - code = textwrap.dedent("""\ - def foo(): - com_text = \\ - ''' - TEST - ''' % (input_fname, output_fname) - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testNoSpacesAroundKeywordDefaultValues(self): - code = textwrap.dedent("""\ - sources = { - 'json': request.get_json(silent=True) or {}, - 'json2': request.get_json(silent=True), - } - json = request.get_json(silent=True) or {} - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testNoSplittingBeforeEndingSubscriptBracket(self): - unformatted_code = textwrap.dedent("""\ - if True: - if True: - status = cf.describe_stacks(StackName=stackname)[u'Stacks'][0][u'StackStatus'] - """) - expected_formatted_code = textwrap.dedent("""\ - if True: - if True: - status = cf.describe_stacks( - StackName=stackname)[u'Stacks'][0][u'StackStatus'] - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testNoSplittingOnSingleArgument(self): - unformatted_code = textwrap.dedent("""\ - xxxxxxxxxxxxxx = (re.search(r'(\\d+\\.\\d+\\.\\d+\\.)\\d+', - aaaaaaa.bbbbbbbbbbbb).group(1) + - re.search(r'\\d+\\.\\d+\\.\\d+\\.(\\d+)', - ccccccc).group(1)) - xxxxxxxxxxxxxx = (re.search(r'(\\d+\\.\\d+\\.\\d+\\.)\\d+', - aaaaaaa.bbbbbbbbbbbb).group(a.b) + - re.search(r'\\d+\\.\\d+\\.\\d+\\.(\\d+)', - ccccccc).group(c.d)) - """) - expected_formatted_code = textwrap.dedent("""\ - xxxxxxxxxxxxxx = ( - re.search(r'(\\d+\\.\\d+\\.\\d+\\.)\\d+', aaaaaaa.bbbbbbbbbbbb).group(1) + - re.search(r'\\d+\\.\\d+\\.\\d+\\.(\\d+)', ccccccc).group(1)) - xxxxxxxxxxxxxx = ( - re.search(r'(\\d+\\.\\d+\\.\\d+\\.)\\d+', aaaaaaa.bbbbbbbbbbbb).group(a.b) + - re.search(r'\\d+\\.\\d+\\.\\d+\\.(\\d+)', ccccccc).group(c.d)) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testSplittingArraysSensibly(self): - unformatted_code = textwrap.dedent("""\ - while True: - while True: - aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa = list['bbbbbbbbbbbbbbbbbbbbbbbbb'].split(',') - aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa = list('bbbbbbbbbbbbbbbbbbbbbbbbb').split(',') - """) - expected_formatted_code = textwrap.dedent("""\ - while True: - while True: - aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa = list[ - 'bbbbbbbbbbbbbbbbbbbbbbbbb'].split(',') - aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa = list( - 'bbbbbbbbbbbbbbbbbbbbbbbbb').split(',') - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testComprehensionForAndIf(self): - unformatted_code = textwrap.dedent("""\ - class f: - - def __repr__(self): - tokens_repr = ','.join(['{0}({1!r})'.format(tok.name, tok.value) for tok in self._tokens]) - """) - expected_formatted_code = textwrap.dedent("""\ - class f: - - def __repr__(self): - tokens_repr = ','.join( - ['{0}({1!r})'.format(tok.name, tok.value) for tok in self._tokens]) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testFunctionCallArguments(self): - unformatted_code = textwrap.dedent("""\ - def f(): - if True: - pytree_utils.InsertNodesBefore(_CreateCommentsFromPrefix( - comment_prefix, comment_lineno, comment_column, - standalone=True), ancestor_at_indent) - pytree_utils.InsertNodesBefore(_CreateCommentsFromPrefix( - comment_prefix, comment_lineno, comment_column, - standalone=True)) - """) - expected_formatted_code = textwrap.dedent("""\ - def f(): - if True: - pytree_utils.InsertNodesBefore( - _CreateCommentsFromPrefix( - comment_prefix, comment_lineno, comment_column, standalone=True), - ancestor_at_indent) - pytree_utils.InsertNodesBefore( - _CreateCommentsFromPrefix( - comment_prefix, comment_lineno, comment_column, standalone=True)) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testBinaryOperators(self): - unformatted_code = textwrap.dedent("""\ - a = b ** 37 - """) - expected_formatted_code = textwrap.dedent("""\ - a = b**37 - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testBinaryOperators(self): - code = textwrap.dedent("""\ - def f(): - if True: - if (self.stack[-1].split_before_closing_bracket and - # FIXME(morbo): Use the 'matching_bracket' instead of this. - # FIXME(morbo): Don't forget about tuples! - current.value in ']}'): - pass - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testContiguousList(self): - code = textwrap.dedent("""\ - [retval1, retval2] = a_very_long_function(argument_1, argument2, argument_3, - argument_4) - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testArgsAndKwargsFormatting(self): - code = textwrap.dedent("""\ - a(a=aaaaaaaaaaaaaaaaaaaaa, - b=aaaaaaaaaaaaaaaaaaaaaaaa, - c=aaaaaaaaaaaaaaaaaa, - *d, - **e) - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testCommentColumnLimitOverflow(self): - code = textwrap.dedent("""\ - def f(): - if True: - TaskManager.get_tags = MagicMock( - name='get_tags_mock', - return_value=[157031694470475], - # side_effect=[(157031694470475), (157031694470475),], - ) - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testMultilineLambdas(self): - try: - style.SetGlobalStyle( - style.CreateStyleFromConfig( - '{based_on_style: chromium, allow_multiline_lambdas: true}')) - unformatted_code = textwrap.dedent("""\ - class SomeClass(object): - do_something = True - - def succeeded(self, dddddddddddddd): - d = defer.succeed(None) - - if self.do_something: - d.addCallback(lambda _: self.aaaaaa.bbbbbbbbbbbbbbbb.cccccccccccccccccccccccccccccccc(dddddddddddddd)) - return d - """) - expected_formatted_code = textwrap.dedent("""\ - class SomeClass(object): - do_something = True - - def succeeded(self, dddddddddddddd): - d = defer.succeed(None) - - if self.do_something: - d.addCallback(lambda _: self.aaaaaa.bbbbbbbbbbbbbbbb. - cccccccccccccccccccccccccccccccc(dddddddddddddd)) - return d - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, - reformatter.Reformat(uwlines)) - finally: - style.SetGlobalStyle(style.CreateChromiumStyle()) - - def testStableDictionaryFormatting(self): - try: - style.SetGlobalStyle( - style.CreateStyleFromConfig( - '{based_on_style: pep8, indent_width: 2, ' - 'continuation_indent_width: 4, indent_dictionary_value: True}')) - code = textwrap.dedent("""\ - class A(object): - def method(self): - filters = { - 'expressions': [ - {'field': { - 'search_field': - {'user_field': 'latest_party__number_of_guests'}, - }} - ] - } - """) - uwlines = _ParseAndUnwrap(code) - reformatted_code = reformatter.Reformat(uwlines) - self.assertCodeEqual(code, reformatted_code) - - uwlines = _ParseAndUnwrap(reformatted_code) - reformatted_code = reformatter.Reformat(uwlines) - self.assertCodeEqual(code, reformatted_code) - finally: - style.SetGlobalStyle(style.CreateChromiumStyle()) - - def testDontSplitKeywordValueArguments(self): - code = textwrap.dedent("""\ - def mark_game_scored(gid): - _connect.execute(_games.update().where(_games.c.gid == gid).values( - scored=True)) - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testDontAddBlankLineAfterMultilineString(self): - code = textwrap.dedent("""\ - query = '''SELECT id - FROM table - WHERE day in {}''' - days = ",".join(days) - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testFormattingListComprehensions(self): - code = textwrap.dedent("""\ - def a(): - if True: - if True: - if True: - columns = [x for x, y in self._heap_this_is_very_long - if x.route[0] == choice] - self._heap = [x for x in self._heap if x.route and x.route[0] == choice] - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testNoSplittingWhenBinPacking(self): - try: - style.SetGlobalStyle( - style.CreateStyleFromConfig( - '{based_on_style: pep8, indent_width: 2, ' - 'continuation_indent_width: 4, indent_dictionary_value: True, ' - 'dedent_closing_brackets: True, ' - 'split_before_named_assigns: False}')) - code = textwrap.dedent("""\ - a_very_long_function_name( - long_argument_name_1=1, - long_argument_name_2=2, - long_argument_name_3=3, - long_argument_name_4=4, - ) - - a_very_long_function_name( - long_argument_name_1=1, long_argument_name_2=2, long_argument_name_3=3, - long_argument_name_4=4 - ) - """) - uwlines = _ParseAndUnwrap(code) - reformatted_code = reformatter.Reformat(uwlines) - self.assertCodeEqual(code, reformatted_code) - - uwlines = _ParseAndUnwrap(reformatted_code) - reformatted_code = reformatter.Reformat(uwlines) - self.assertCodeEqual(code, reformatted_code) - finally: - style.SetGlobalStyle(style.CreateChromiumStyle()) - - def testNotSplittingAfterSubscript(self): - unformatted_code = textwrap.dedent("""\ - if not aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.b(c == d[ - 'eeeeee']).ffffff(): - pass - """) - expected_formatted_code = textwrap.dedent("""\ - if not aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.b( - c == d['eeeeee']).ffffff(): - pass - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testSplittingOneArgumentList(self): - unformatted_code = textwrap.dedent("""\ - def _(): - if True: - if True: - if True: - if True: - if True: - boxes[id_] = np.concatenate((points.min(axis=0), qoints.max(axis=0))) - """) - expected_formatted_code = textwrap.dedent("""\ - def _(): - if True: - if True: - if True: - if True: - if True: - boxes[id_] = np.concatenate( - (points.min(axis=0), qoints.max(axis=0))) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testSplittingBeforeFirstElementListArgument(self): - unformatted_code = textwrap.dedent("""\ - class _(): - @classmethod - def _pack_results_for_constraint_or(cls, combination, constraints): - if True: - if True: - if True: - return cls._create_investigation_result( - ( - clue for clue in combination if not clue == Verifier.UNMATCHED - ), constraints, InvestigationResult.OR - ) - """) - expected_formatted_code = textwrap.dedent("""\ - class _(): - - @classmethod - def _pack_results_for_constraint_or(cls, combination, constraints): - if True: - if True: - if True: - return cls._create_investigation_result( - (clue for clue in combination if not clue == Verifier.UNMATCHED), - constraints, InvestigationResult.OR) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testSplittingArgumentsTerminatedByComma(self): - try: - style.SetGlobalStyle( - style.CreateStyleFromConfig( - '{based_on_style: chromium, ' - 'split_arguments_when_comma_terminated: True}')) - code = textwrap.dedent("""\ - function_name(argument_name_1=1, argument_name_2=2, argument_name_3=3) - - function_name( - argument_name_1=1, - argument_name_2=2, - argument_name_3=3, - ) - - a_very_long_function_name( - long_argument_name_1=1, - long_argument_name_2=2, - long_argument_name_3=3, - long_argument_name_4=4) - - a_very_long_function_name( - long_argument_name_1, - long_argument_name_2, - long_argument_name_3, - long_argument_name_4, - ) - """) - uwlines = _ParseAndUnwrap(code) - reformatted_code = reformatter.Reformat(uwlines) - self.assertCodeEqual(code, reformatted_code) - - uwlines = _ParseAndUnwrap(reformatted_code) - reformatted_code = reformatter.Reformat(uwlines) - self.assertCodeEqual(code, reformatted_code) - finally: - style.SetGlobalStyle(style.CreateChromiumStyle()) - - def testImportAsList(self): - code = textwrap.dedent("""\ - from toto import titi, tata, tutu # noqa - from toto import titi, tata, tutu - from toto import (titi, tata, tutu) - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - -class BuganizerFixes(ReformatterTest): - - @classmethod - def setUpClass(cls): - style.SetGlobalStyle(style.CreateChromiumStyle()) - - def testB30442148(self): - unformatted_code = textwrap.dedent("""\ - def lulz(): - return (some_long_module_name.SomeLongClassName. - some_long_attribute_name.some_long_method_name()) - """) - expected_formatted_code = textwrap.dedent("""\ - def lulz(): - return (some_long_module_name.SomeLongClassName.some_long_attribute_name. - some_long_method_name()) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testB26868213(self): - unformatted_code = textwrap.dedent("""\ - def _(): - xxxxxxxxxxxxxxxxxxx = { - 'ssssss': {'ddddd': 'qqqqq', - 'p90': aaaaaaaaaaaaaaaaa, - 'p99': bbbbbbbbbbbbbbbbb, - 'lllllllllllll': yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy(),}, - 'bbbbbbbbbbbbbbbbbbbbbbbbbbbb': { - 'ddddd': 'bork bork bork bo', - 'p90': wwwwwwwwwwwwwwwww, - 'p99': wwwwwwwwwwwwwwwww, - 'lllllllllllll': None, # use the default - } - } - """) - expected_formatted_code = textwrap.dedent("""\ - def _(): - xxxxxxxxxxxxxxxxxxx = { - 'ssssss': { - 'ddddd': 'qqqqq', - 'p90': aaaaaaaaaaaaaaaaa, - 'p99': bbbbbbbbbbbbbbbbb, - 'lllllllllllll': yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy(), - }, - 'bbbbbbbbbbbbbbbbbbbbbbbbbbbb': { - 'ddddd': 'bork bork bork bo', - 'p90': wwwwwwwwwwwwwwwww, - 'p99': wwwwwwwwwwwwwwwww, - 'lllllllllllll': None, # use the default - } - } - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testB30173198(self): - code = textwrap.dedent("""\ - class _(): - - def _(): - self.assertFalse( - evaluation_runner.get_larps_in_eval_set('these_arent_the_larps')) - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testB29908765(self): - code = textwrap.dedent("""\ - class _(): - - def __repr__(self): - return '' % (self._id, - self._stub._stub.rpc_channel().target()) # pylint:disable=protected-access - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testB30087362(self): - code = textwrap.dedent("""\ - def _(): - for s in sorted(env['foo']): - bar() - # This is a comment - - # This is another comment - foo() - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testB29093579(self): - unformatted_code = textwrap.dedent("""\ - def _(): - _xxxxxxxxxxxxxxx(aaaaaaaa, bbbbbbbbbbbbbb.cccccccccc[ - dddddddddddddddddddddddddddd.eeeeeeeeeeeeeeeeeeeeee.fffffffffffffffffffff]) - """) - expected_formatted_code = textwrap.dedent("""\ - def _(): - _xxxxxxxxxxxxxxx( - aaaaaaaa, - bbbbbbbbbbbbbb.cccccccccc[dddddddddddddddddddddddddddd. - eeeeeeeeeeeeeeeeeeeeee.fffffffffffffffffffff]) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testB26382315(self): - code = textwrap.dedent("""\ - @hello_world - # This is a first comment - - # Comment - def foo(): - pass - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testB27616132(self): - unformatted_code = textwrap.dedent("""\ - if True: - query.fetch_page.assert_has_calls([ - mock.call(100, - start_cursor=None), - mock.call(100, - start_cursor=cursor_1), - mock.call(100, - start_cursor=cursor_2), - ]) - """) - expected_formatted_code = textwrap.dedent("""\ - if True: - query.fetch_page.assert_has_calls([ - mock.call( - 100, start_cursor=None), - mock.call( - 100, start_cursor=cursor_1), - mock.call( - 100, start_cursor=cursor_2), - ]) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testB27590179(self): - unformatted_code = textwrap.dedent("""\ - if True: - if True: - self.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa = ( - { True: - self.bbb.cccccccccc(ddddddddddddddddddddddd.eeeeeeeeeeeeeeeeeeeeee), - False: - self.bbb.cccccccccc(ddddddddddddddddddddddd.eeeeeeeeeeeeeeeeeeeeee) - }) - """) - expected_formatted_code = textwrap.dedent("""\ - if True: - if True: - self.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa = ({ - True: - self.bbb.cccccccccc(ddddddddddddddddddddddd.eeeeeeeeeeeeeeeeeeeeee), - False: - self.bbb.cccccccccc(ddddddddddddddddddddddd.eeeeeeeeeeeeeeeeeeeeee) - }) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testB27266946(self): - unformatted_code = textwrap.dedent("""\ - def _(): - aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa = (self.bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.cccccccccccccccccccccccccccccccccccc) - """) - expected_formatted_code = textwrap.dedent("""\ - def _(): - aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa = ( - self.bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb. - cccccccccccccccccccccccccccccccccccc) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testB25505359(self): - code = textwrap.dedent("""\ - _EXAMPLE = { - 'aaaaaaaaaaaaaa': [ - {'bbbb': 'cccccccccccccccccccccc', - 'dddddddddddd': [ - ]}, {'bbbb': 'ccccccccccccccccccc', - 'dddddddddddd': [ - ]} - ] - } - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testB25324261(self): - code = textwrap.dedent("""\ - aaaaaaaaa = set(bbbb.cccc - for ddd in eeeeee.fffffffffff.gggggggggggggggg - for cccc in ddd.specification) - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testB25136704(self): - code = textwrap.dedent("""\ - class f: - - def test(self): - self.bbbbbbb[0]['aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', - {'xxxxxx': 'yyyyyy'}] = cccccc.ddd('1m', '10x1+1') - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testB25165602(self): - code = textwrap.dedent("""\ - def f(): - ids = {u: i for u, i in zip(self.aaaaa, xrange(42, 42 + len(self.aaaaaa)))} - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testB25157123(self): - code = textwrap.dedent("""\ - def ListArgs(): - FairlyLongMethodName([relatively_long_identifier_for_a_list], - another_argument_with_a_long_identifier) - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testB25136820(self): - unformatted_code = textwrap.dedent("""\ - def foo(): - return collections.OrderedDict({ - # Preceding comment. - 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa': - '$bbbbbbbbbbbbbbbbbbbbbbbb', - }) - """) - expected_formatted_code = textwrap.dedent("""\ - def foo(): - return collections.OrderedDict({ - # Preceding comment. - 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa': - '$bbbbbbbbbbbbbbbbbbbbbbbb', - }) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testB25131481(self): - unformatted_code = textwrap.dedent("""\ - APPARENT_ACTIONS = ('command_type', { - 'materialize': lambda x: some_type_of_function('materialize ' + x.command_def), - '#': lambda x: x # do nothing - }) - """) - expected_formatted_code = textwrap.dedent("""\ - APPARENT_ACTIONS = ( - 'command_type', - { - 'materialize': - lambda x: some_type_of_function('materialize ' + x.command_def), - '#': lambda x: x # do nothing - }) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testB23445244(self): - unformatted_code = textwrap.dedent("""\ - def foo(): - if True: - return xxxxxxxxxxxxxxxx( - command, - extra_env={ - "OOOOOOOOOOOOOOOOOOOOO": FLAGS.zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz, - "PPPPPPPPPPPPPPPPPPPPP": - FLAGS.aaaaaaaaaaaaaa + FLAGS.bbbbbbbbbbbbbbbbbbb, - }) - """) - expected_formatted_code = textwrap.dedent("""\ - def foo(): - if True: - return xxxxxxxxxxxxxxxx( - command, - extra_env={ - "OOOOOOOOOOOOOOOOOOOOO": - FLAGS.zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz, - "PPPPPPPPPPPPPPPPPPPPP": FLAGS.aaaaaaaaaaaaaa + - FLAGS.bbbbbbbbbbbbbbbbbbb, - }) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testB20559654(self): - unformatted_code = textwrap.dedent("""\ - class A(object): - - def foo(self): - unused_error, result = server.Query( - ['AA BBBB CCC DDD EEEEEEEE X YY ZZZZ FFF EEE AAAAAAAA'], - aaaaaaaaaaa=True, bbbbbbbb=None) - """) - expected_formatted_code = textwrap.dedent("""\ - class A(object): - - def foo(self): - unused_error, result = server.Query( - ['AA BBBB CCC DDD EEEEEEEE X YY ZZZZ FFF EEE AAAAAAAA'], - aaaaaaaaaaa=True, - bbbbbbbb=None) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testB23943842(self): - unformatted_code = textwrap.dedent("""\ - class F(): - def f(): - self.assertDictEqual( - accounts, { - 'foo': - {'account': 'foo', - 'lines': 'l1\\nl2\\nl3\\n1 line(s) were elided.'}, - 'bar': {'account': 'bar', - 'lines': 'l5\\nl6\\nl7'}, - 'wiz': {'account': 'wiz', - 'lines': 'l8'} - }) - """) - expected_formatted_code = textwrap.dedent("""\ - class F(): - - def f(): - self.assertDictEqual(accounts, { - 'foo': {'account': 'foo', - 'lines': 'l1\\nl2\\nl3\\n1 line(s) were elided.'}, - 'bar': {'account': 'bar', - 'lines': 'l5\\nl6\\nl7'}, - 'wiz': {'account': 'wiz', - 'lines': 'l8'} - }) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testB20551180(self): - unformatted_code = textwrap.dedent("""\ - def foo(): - if True: - return (struct.pack('aaaa', bbbbbbbbbb, ccccccccccccccc, dddddddd) + eeeeeee) - """) - expected_formatted_code = textwrap.dedent("""\ - def foo(): - if True: - return ( - struct.pack('aaaa', bbbbbbbbbb, ccccccccccccccc, dddddddd) + eeeeeee) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testB23944849(self): - unformatted_code = textwrap.dedent("""\ - class A(object): - def xxxxxxxxx(self, aaaaaaa, bbbbbbb=ccccccccccc, dddddd=300, eeeeeeeeeeeeee=None, fffffffffffffff=0): - pass - """) - expected_formatted_code = textwrap.dedent("""\ - class A(object): - - def xxxxxxxxx(self, - aaaaaaa, - bbbbbbb=ccccccccccc, - dddddd=300, - eeeeeeeeeeeeee=None, - fffffffffffffff=0): - pass - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testB23935890(self): - unformatted_code = textwrap.dedent("""\ - class F(): - def functioni(self, aaaaaaa, bbbbbbb, cccccc, dddddddddddddd, eeeeeeeeeeeeeee): - pass - """) - expected_formatted_code = textwrap.dedent("""\ - class F(): - - def functioni(self, aaaaaaa, bbbbbbb, cccccc, dddddddddddddd, - eeeeeeeeeeeeeee): - pass - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testB28414371(self): - code = textwrap.dedent("""\ - def _(): - return ( - (m.fffff( - m.rrr('mmmmmmmmmmmmmmmm', 'ssssssssssssssssssssssssss'), - ffffffffffffffff) - | m.wwwwww(m.ddddd('1h')) - | m.ggggggg(bbbbbbbbbbbbbbb) - | m.ppppp( - (1 - m.ffffffffffffffff(llllllllllllllllllllll * 1000000, m.vvv)) * - m.ddddddddddddddddd(m.vvv)), m.fffff( - m.rrr('mmmmmmmmmmmmmmmm', 'sssssssssssssssssssssss'), - dict( - ffffffffffffffff, - **{ - 'mmmmmm:ssssss': m.rrrrrrrrrrr( - '|'.join(iiiiiiiiiiiiii), iiiiii=True) - })) - | m.wwwwww(m.rrrr('1h')) - | m.ggggggg(bbbbbbbbbbbbbbb)) - | m.jjjj() - | m.ppppp(m.vvv[0] + m.vvv[1])) - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testB20127686(self): - code = textwrap.dedent("""\ - def f(): - if True: - return ((m.fffff( - m.rrr('xxxxxxxxxxxxxxxx', - 'yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy'), - mmmmmmmm) - | m.wwwwww(m.rrrr(self.tttttttttt, self.mmmmmmmmmmmmmmmmmmmmm)) - | m.ggggggg(self.gggggggg, m.sss()), m.fffff('aaaaaaaaaaaaaaaa') - | m.wwwwww(m.ddddd(self.tttttttttt, self.mmmmmmmmmmmmmmmmmmmmm)) - | m.ggggggg(self.gggggggg)) - | m.jjjj() - | m.ppppp(m.VAL[0] / m.VAL[1])) - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testB20016122(self): - try: - style.SetGlobalStyle( - style.CreateStyleFromConfig('{based_on_style: chromium, ' - 'SPLIT_BEFORE_LOGICAL_OPERATOR: True}')) - code = textwrap.dedent("""\ - class foo(): - - def __eq__(self, other): - return (isinstance(other, type(self)) - and self.xxxxxxxxxxx == other.xxxxxxxxxxx - and self.xxxxxxxx == other.xxxxxxxx - and self.aaaaaaaaaaaa == other.aaaaaaaaaaaa - and self.bbbbbbbbbbb == other.bbbbbbbbbbb - and self.ccccccccccccccccc == other.ccccccccccccccccc - and self.ddddddddddddddddddddddd == other.ddddddddddddddddddddddd - and self.eeeeeeeeeeee == other.eeeeeeeeeeee - and self.ffffffffffffff == other.time_completed - and self.gggggg == other.gggggg and self.hhh == other.hhh - and len(self.iiiiiiii) == len(other.iiiiiiii) - and all(jjjjjjj in other.iiiiiiii for jjjjjjj in self.iiiiiiii)) - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - finally: - style.SetGlobalStyle(style.CreateChromiumStyle()) - - def testB22527411(self): - unformatted_code = textwrap.dedent("""\ - def f(): - if True: - aaaaaa.bbbbbbbbbbbbbbbbbbbb[-1].cccccccccccccc.ddd().eeeeeeee(ffffffffffffff) - """) - expected_formatted_code = textwrap.dedent("""\ - def f(): - if True: - aaaaaa.bbbbbbbbbbbbbbbbbbbb[-1].cccccccccccccc.ddd().eeeeeeee( - ffffffffffffff) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testB20849933(self): - code = textwrap.dedent("""\ - def main(unused_argv): - if True: - aaaaaaaa = { - 'xxx': '%s/cccccc/ddddddddddddddddddd.jar' % - (eeeeee.FFFFFFFFFFFFFFFFFF), - } - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testB20813997(self): - code = textwrap.dedent("""\ - def myfunc_1(): - myarray = numpy.zeros((2, 2, 2)) - print(myarray[:, 1, :]) - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testB20605036(self): - code = textwrap.dedent("""\ - foo = { - 'aaaa': { - # A comment for no particular reason. - 'xxxxxxxx': 'bbbbbbbbb', - 'yyyyyyyyyyyyyyyyyy': 'cccccccccccccccccccccccccccccc' - 'dddddddddddddddddddddddddddddddddddddddddd', - } - } - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testB20562732(self): - code = textwrap.dedent("""\ - foo = [ - # Comment about first list item - 'First item', - # Comment about second list item - 'Second item', - ] - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testB20128830(self): - code = textwrap.dedent("""\ - a = { - 'xxxxxxxxxxxxxxxxxxxx': { - 'aaaa': 'mmmmmmm', - 'bbbbb': 'mmmmmmmmmmmmmmmmmmmmm', - 'cccccccccc': [ - 'nnnnnnnnnnn', - 'ooooooooooo', - 'ppppppppppp', - 'qqqqqqqqqqq', - ], - }, - } - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testB20073838(self): - code = textwrap.dedent("""\ - class DummyModel(object): - - def do_nothing(self, class_1_count): - if True: - class_0_count = num_votes - class_1_count - return ('{class_0_name}={class_0_count}, {class_1_name}={class_1_count}' - .format( - class_0_name=self.class_0_name, - class_0_count=class_0_count, - class_1_name=self.class_1_name, - class_1_count=class_1_count)) - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testB19626808(self): - code = textwrap.dedent("""\ - if True: - aaaaaaaaaaaaaaaaaaaaaaa.bbbbbbbbb( - 'ccccccccccc', ddddddddd='eeeee').fffffffff([ - ggggggggggggggggggggg - ]) - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testB19547210(self): - code = textwrap.dedent("""\ - while True: - if True: - if True: - if True: - if xxxxxxxxxxxx.yyyyyyy(aa).zzzzzzz() not in ( - xxxxxxxxxxxx.yyyyyyyyyyyyyy.zzzzzzzz, - xxxxxxxxxxxx.yyyyyyyyyyyyyy.zzzzzzzz): - continue - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testB19377034(self): - code = textwrap.dedent("""\ - def f(): - if (aaaaaaaaaaaaaaa.start >= aaaaaaaaaaaaaaa.end or - bbbbbbbbbbbbbbb.start >= bbbbbbbbbbbbbbb.end): - return False - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testB19372573(self): - code = textwrap.dedent("""\ - def f(): - if a: return 42 - while True: - if b: continue - if c: break - return 0 - """) - uwlines = _ParseAndUnwrap(code) - try: - style.SetGlobalStyle(style.CreatePEP8Style()) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - finally: - style.SetGlobalStyle(style.CreateChromiumStyle()) - - def testB19353268(self): - code = textwrap.dedent("""\ - a = {1, 2, 3}[x] - b = {'foo': 42, 'bar': 37}['foo'] - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testB19287512(self): - unformatted_code = textwrap.dedent("""\ - class Foo(object): - - def bar(self): - with xxxxxxxxxx.yyyyy( - 'aaaaaaa.bbbbbbbb.ccccccc.dddddddddddddddddddd.eeeeeeeeeee', - fffffffffff=(aaaaaaa.bbbbbbbb.ccccccc.dddddddddddddddddddd - .Mmmmmmmmmmmmmmmmmm(-1, 'permission error'))): - self.assertRaises(nnnnnnnnnnnnnnnn.ooooo, ppppp.qqqqqqqqqqqqqqqqq) - """) - expected_formatted_code = textwrap.dedent("""\ - class Foo(object): - - def bar(self): - with xxxxxxxxxx.yyyyy( - 'aaaaaaa.bbbbbbbb.ccccccc.dddddddddddddddddddd.eeeeeeeeeee', - fffffffffff=( - aaaaaaa.bbbbbbbb.ccccccc.dddddddddddddddddddd.Mmmmmmmmmmmmmmmmmm( - -1, 'permission error'))): - self.assertRaises(nnnnnnnnnnnnnnnn.ooooo, ppppp.qqqqqqqqqqqqqqqqq) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testB19194420(self): - code = textwrap.dedent("""\ - method.Set( - 'long argument goes here that causes the line to break', - lambda arg2=0.5: arg2) - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testB19073499(self): - code = textwrap.dedent("""\ - instance = (aaaaaaa.bbbbbbb().ccccccccccccccccc().ddddddddddd({ - 'aa': 'context!' - }).eeeeeeeeeeeeeeeeeee({ # Inline comment about why fnord has the value 6. - 'fnord': 6 - })) - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testB18257115(self): - code = textwrap.dedent("""\ - if True: - if True: - self._Test(aaaa, bbbbbbb.cccccccccc, dddddddd, eeeeeeeeeee, - [ffff, ggggggggggg, hhhhhhhhhhhh, iiiiii, jjjj]) - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testB18256666(self): - code = textwrap.dedent("""\ - class Foo(object): - - def Bar(self): - aaaaa.bbbbbbb( - ccc='ddddddddddddddd', - eeee='ffffffffffffffffffffff-%s-%s' % (gggg, int(time.time())), - hhhhhh={ - 'iiiiiiiiiii': iiiiiiiiiii, - 'jjjj': jjjj.jjjjj(), - 'kkkkkkkkkkkk': kkkkkkkkkkkk, - }, - llllllllll=mmmmmm.nnnnnnnnnnnnnnnn) - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testB18256826(self): - code = textwrap.dedent("""\ - if True: - pass - # A multiline comment. - # Line two. - elif False: - pass - - if True: - pass - # A multiline comment. - # Line two. - elif False: - pass - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testB18255697(self): - code = textwrap.dedent("""\ - AAAAAAAAAAAAAAA = { - 'XXXXXXXXXXXXXX': 4242, # Inline comment - # Next comment - 'YYYYYYYYYYYYYYYY': ['zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz'], - } - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testB17534869(self): - unformatted_code = textwrap.dedent("""\ - if True: - self.assertLess(abs(time.time()-aaaa.bbbbbbbbbbb( - datetime.datetime.now())), 1) - """) - expected_formatted_code = textwrap.dedent("""\ - if True: - self.assertLess( - abs(time.time() - aaaa.bbbbbbbbbbb(datetime.datetime.now())), 1) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testB17489866(self): - unformatted_code = textwrap.dedent("""\ - def f(): - if True: - if True: - return aaaa.bbbbbbbbb(ccccccc=dddddddddddddd({('eeee', \ -'ffffffff'): str(j)})) - """) - expected_formatted_code = textwrap.dedent("""\ - def f(): - if True: - if True: - return aaaa.bbbbbbbbb(ccccccc=dddddddddddddd( - {('eeee', 'ffffffff'): str(j)})) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testB17133019(self): - unformatted_code = textwrap.dedent("""\ - class aaaaaaaaaaaaaa(object): - - def bbbbbbbbbb(self): - with io.open("/dev/null", "rb"): - with io.open(os.path.join(aaaaa.bbbbb.ccccccccccc, - DDDDDDDDDDDDDDD, - "eeeeeeeee ffffffffff" - ), "rb") as gggggggggggggggggggg: - print(gggggggggggggggggggg) - """) - expected_formatted_code = textwrap.dedent("""\ - class aaaaaaaaaaaaaa(object): - - def bbbbbbbbbb(self): - with io.open("/dev/null", "rb"): - with io.open( - os.path.join(aaaaa.bbbbb.ccccccccccc, DDDDDDDDDDDDDDD, - "eeeeeeeee ffffffffff"), "rb") as gggggggggggggggggggg: - print(gggggggggggggggggggg) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testB17011869(self): - unformatted_code = textwrap.dedent("""\ - '''blah......''' - - class SomeClass(object): - '''blah.''' - - AAAAAAAAAAAA = { # Comment. - 'BBB': 1.0, - 'DDDDDDDD': 0.4811 - } - """) - expected_formatted_code = textwrap.dedent("""\ - '''blah......''' - - - class SomeClass(object): - '''blah.''' - - AAAAAAAAAAAA = { # Comment. - 'BBB': 1.0, - 'DDDDDDDD': 0.4811 - } - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testB16783631(self): - unformatted_code = textwrap.dedent("""\ - if True: - with aaaaaaaaaaaaaa.bbbbbbbbbbbbb.ccccccc(ddddddddddddd, - eeeeeeeee=self.fffffffffffff - )as gggg: - pass - """) - expected_formatted_code = textwrap.dedent("""\ - if True: - with aaaaaaaaaaaaaa.bbbbbbbbbbbbb.ccccccc( - ddddddddddddd, eeeeeeeee=self.fffffffffffff) as gggg: - pass - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testB16572361(self): - unformatted_code = textwrap.dedent("""\ - def foo(self): - def bar(my_dict_name): - self.my_dict_name['foo-bar-baz-biz-boo-baa-baa'].IncrementBy.assert_called_once_with('foo_bar_baz_boo') - """) - expected_formatted_code = textwrap.dedent("""\ - def foo(self): - - def bar(my_dict_name): - self.my_dict_name[ - 'foo-bar-baz-biz-boo-baa-baa'].IncrementBy.assert_called_once_with( - 'foo_bar_baz_boo') - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testB15884241(self): - unformatted_code = textwrap.dedent("""\ - if 1: - if 1: - for row in AAAA: - self.create(aaaaaaaa="/aaa/bbbb/cccc/dddddd/eeeeeeeeeeeeeeeeeeeeeeeeee/%s" % row [0].replace(".foo", ".bar"), aaaaa=bbb[1], ccccc=bbb[2], dddd=bbb[3], eeeeeeeeeee=[s.strip() for s in bbb[4].split(",")], ffffffff=[s.strip() for s in bbb[5].split(",")], gggggg=bbb[6]) - """) - expected_formatted_code = textwrap.dedent("""\ - if 1: - if 1: - for row in AAAA: - self.create( - aaaaaaaa="/aaa/bbbb/cccc/dddddd/eeeeeeeeeeeeeeeeeeeeeeeeee/%s" % - row[0].replace(".foo", ".bar"), - aaaaa=bbb[1], - ccccc=bbb[2], - dddd=bbb[3], - eeeeeeeeeee=[s.strip() for s in bbb[4].split(",")], - ffffffff=[s.strip() for s in bbb[5].split(",")], - gggggg=bbb[6]) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testB15697268(self): - unformatted_code = textwrap.dedent("""\ - def main(unused_argv): - ARBITRARY_CONSTANT_A = 10 - an_array_with_an_exceedingly_long_name = range(ARBITRARY_CONSTANT_A + 1) - ok = an_array_with_an_exceedingly_long_name[:ARBITRARY_CONSTANT_A] - bad_slice = map(math.sqrt, an_array_with_an_exceedingly_long_name[:ARBITRARY_CONSTANT_A]) - a_long_name_slicing = an_array_with_an_exceedingly_long_name[:ARBITRARY_CONSTANT_A] - bad_slice = ("I am a crazy, no good, string whats too long, etc." + " no really ")[:ARBITRARY_CONSTANT_A] - """) - expected_formatted_code = textwrap.dedent("""\ - def main(unused_argv): - ARBITRARY_CONSTANT_A = 10 - an_array_with_an_exceedingly_long_name = range(ARBITRARY_CONSTANT_A + 1) - ok = an_array_with_an_exceedingly_long_name[:ARBITRARY_CONSTANT_A] - bad_slice = map(math.sqrt, - an_array_with_an_exceedingly_long_name[:ARBITRARY_CONSTANT_A]) - a_long_name_slicing = an_array_with_an_exceedingly_long_name[: - ARBITRARY_CONSTANT_A] - bad_slice = ("I am a crazy, no good, string whats too long, etc." + - " no really ")[:ARBITRARY_CONSTANT_A] - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testB15597568(self): - unformatted_code = textwrap.dedent("""\ - if True: - if True: - if True: - print(("Return code was %d" + (", and the process timed out." if did_time_out else ".")) % errorcode) - """) - expected_formatted_code = textwrap.dedent("""\ - if True: - if True: - if True: - print(("Return code was %d" + (", and the process timed out." - if did_time_out else ".")) % errorcode) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testB15542157(self): - unformatted_code = textwrap.dedent("""\ - aaaaaaaaaaaa = bbbb.ccccccccccccccc(dddddd.eeeeeeeeeeeeee, ffffffffffffffffff, gggggg.hhhhhhhhhhhhhhhhh) - """) - expected_formatted_code = textwrap.dedent("""\ - aaaaaaaaaaaa = bbbb.ccccccccccccccc(dddddd.eeeeeeeeeeeeee, ffffffffffffffffff, - gggggg.hhhhhhhhhhhhhhhhh) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testB15438132(self): - unformatted_code = textwrap.dedent("""\ - if aaaaaaa.bbbbbbbbbb: - cccccc.dddddddddd(eeeeeeeeeee=fffffffffffff.gggggggggggggggggg) - if hhhhhh.iiiii.jjjjjjjjjjjjj: - # This is a comment in the middle of it all. - kkkkkkk.llllllllll.mmmmmmmmmmmmm = True - if (aaaaaa.bbbbb.ccccccccccccc != ddddddd.eeeeeeeeee.fffffffffffff or - eeeeee.fffff.ggggggggggggggggggggggggggg() != hhhhhhh.iiiiiiiiii.jjjjjjjjjjjj): - aaaaaaaa.bbbbbbbbbbbb( - aaaaaa.bbbbb.cc, - dddddddddddd=eeeeeeeeeeeeeeeeeee.fffffffffffffffff( - gggggg.hh, - iiiiiiiiiiiiiiiiiii.jjjjjjjjjj.kkkkkkk, - lllll.mm), - nnnnnnnnnn=ooooooo.pppppppppp) - """) - expected_formatted_code = textwrap.dedent("""\ - if aaaaaaa.bbbbbbbbbb: - cccccc.dddddddddd(eeeeeeeeeee=fffffffffffff.gggggggggggggggggg) - if hhhhhh.iiiii.jjjjjjjjjjjjj: - # This is a comment in the middle of it all. - kkkkkkk.llllllllll.mmmmmmmmmmmmm = True - if (aaaaaa.bbbbb.ccccccccccccc != ddddddd.eeeeeeeeee.fffffffffffff or - eeeeee.fffff.ggggggggggggggggggggggggggg() != - hhhhhhh.iiiiiiiiii.jjjjjjjjjjjj): - aaaaaaaa.bbbbbbbbbbbb( - aaaaaa.bbbbb.cc, - dddddddddddd=eeeeeeeeeeeeeeeeeee.fffffffffffffffff( - gggggg.hh, iiiiiiiiiiiiiiiiiii.jjjjjjjjjj.kkkkkkk, lllll.mm), - nnnnnnnnnn=ooooooo.pppppppppp) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testB14468247(self): - unformatted_code = textwrap.dedent("""\ - call(a=1, - b=2, - ) - """) - expected_formatted_code = textwrap.dedent("""\ - call( - a=1, - b=2,) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testB14406499(self): - unformatted_code = textwrap.dedent("""\ - def foo1(parameter_1, parameter_2, parameter_3, parameter_4, \ -parameter_5, parameter_6): pass - """) - expected_formatted_code = textwrap.dedent("""\ - def foo1(parameter_1, parameter_2, parameter_3, parameter_4, parameter_5, - parameter_6): - pass - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testB13900309(self): - unformatted_code = textwrap.dedent("""\ - self.aaaaaaaaaaa( # A comment in the middle of it all. - 948.0/3600, self.bbb.ccccccccccccccccccccc(dddddddddddddddd.eeee, True)) - """) - expected_formatted_code = textwrap.dedent("""\ - self.aaaaaaaaaaa( # A comment in the middle of it all. - 948.0 / 3600, self.bbb.ccccccccccccccccccccc(dddddddddddddddd.eeee, True)) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - code = textwrap.dedent("""\ - aaaaaaaaaa.bbbbbbbbbbbbbbbbbbbbbbbb.cccccccccccccccccccccccccccccc( - DC_1, (CL - 50, CL), AAAAAAAA, BBBBBBBBBBBBBBBB, 98.0, - CCCCCCC).ddddddddd( # Look! A comment is here. - AAAAAAAA - (20 * 60 - 5)) - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - unformatted_code = textwrap.dedent("""\ - aaaaaaaaaaaaaaaaaaaaaaaa.bbbbbbbbbbbbb.ccccccccccccccccccccccccc().dddddddddddddddddddddddddd(1, 2, 3, 4) - """) - expected_formatted_code = textwrap.dedent("""\ - aaaaaaaaaaaaaaaaaaaaaaaa.bbbbbbbbbbbbb.ccccccccccccccccccccccccc( - ).dddddddddddddddddddddddddd(1, 2, 3, 4) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - unformatted_code = textwrap.dedent("""\ - aaaaaaaaaaaaaaaaaaaaaaaa.bbbbbbbbbbbbb.ccccccccccccccccccccccccc(x).dddddddddddddddddddddddddd(1, 2, 3, 4) - """) - expected_formatted_code = textwrap.dedent("""\ - aaaaaaaaaaaaaaaaaaaaaaaa.bbbbbbbbbbbbb.ccccccccccccccccccccccccc( - x).dddddddddddddddddddddddddd(1, 2, 3, 4) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - unformatted_code = textwrap.dedent("""\ - aaaaaaaaaaaaaaaaaaaaaaaa(xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx).dddddddddddddddddddddddddd(1, 2, 3, 4) - """) - expected_formatted_code = textwrap.dedent("""\ - aaaaaaaaaaaaaaaaaaaaaaaa( - xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx).dddddddddddddddddddddddddd(1, 2, 3, 4) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - unformatted_code = textwrap.dedent("""\ - aaaaaaaaaaaaaaaaaaaaaaaa().bbbbbbbbbbbbbbbbbbbbbbbb().ccccccccccccccccccc().\ -dddddddddddddddddd().eeeeeeeeeeeeeeeeeeeee().fffffffffffffffff().gggggggggggggggggg() - """) - expected_formatted_code = textwrap.dedent("""\ - aaaaaaaaaaaaaaaaaaaaaaaa().bbbbbbbbbbbbbbbbbbbbbbbb().ccccccccccccccccccc( - ).dddddddddddddddddd().eeeeeeeeeeeeeeeeeeeee().fffffffffffffffff( - ).gggggggggggggggggg() - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - -class TestsForPEP8Style(ReformatterTest): - - @classmethod - def setUpClass(cls): - style.SetGlobalStyle(style.CreatePEP8Style()) - - def testIndent4(self): - unformatted_code = textwrap.dedent("""\ - if a+b: - pass - """) - expected_formatted_code = textwrap.dedent("""\ - if a + b: - pass - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testSingleLineIfStatements(self): - code = textwrap.dedent("""\ - if True: a = 42 - elif False: b = 42 - else: c = 42 - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testNoBlankBetweenClassAndDef(self): - unformatted_code = textwrap.dedent("""\ - class Foo: - - def joe(): - pass - """) - expected_formatted_code = textwrap.dedent("""\ - class Foo: - def joe(): - pass - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testSingleWhiteBeforeTrailingComment(self): - unformatted_code = textwrap.dedent("""\ - if a+b: # comment - pass - """) - expected_formatted_code = textwrap.dedent("""\ - if a + b: # comment - pass - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testSplittingSemicolonStatements(self): - unformatted_code = textwrap.dedent("""\ - def f(): - x = y + 42 ; z = n * 42 - if True: a += 1 ; b += 1; c += 1 - """) - expected_formatted_code = textwrap.dedent("""\ - def f(): - x = y + 42 - z = n * 42 - if True: - a += 1 - b += 1 - c += 1 - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testSpaceBetweenEndingCommandAndClosingBracket(self): - unformatted_code = textwrap.dedent("""\ - a = [ - 1, - ] - """) - expected_formatted_code = textwrap.dedent("""\ - a = [1, ] - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testContinuedNonOudentedLine(self): - code = textwrap.dedent("""\ - class eld(d): - if str(geom.geom_type).upper( - ) != self.geom_type and not self.geom_type == 'GEOMETRY': - ror(code='om_type') - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines)) - - def testWrappingPercentExpressions(self): - unformatted_code = textwrap.dedent("""\ - def f(): - if True: - zzzzz = '%s-%s' % (xxxxxxxxxxxxxxxxxxxxxxxxxx + 1, xxxxxxxxxxxxxxxxx.yyy + 1) - zzzzz = '%s-%s'.ww(xxxxxxxxxxxxxxxxxxxxxxxxxx + 1, xxxxxxxxxxxxxxxxx.yyy + 1) - zzzzz = '%s-%s' % (xxxxxxxxxxxxxxxxxxxxxxx + 1, xxxxxxxxxxxxxxxxxxxxx + 1) - zzzzz = '%s-%s'.ww(xxxxxxxxxxxxxxxxxxxxxxx + 1, xxxxxxxxxxxxxxxxxxxxx + 1) - """) - expected_formatted_code = textwrap.dedent("""\ - def f(): - if True: - zzzzz = '%s-%s' % (xxxxxxxxxxxxxxxxxxxxxxxxxx + 1, - xxxxxxxxxxxxxxxxx.yyy + 1) - zzzzz = '%s-%s'.ww(xxxxxxxxxxxxxxxxxxxxxxxxxx + 1, - xxxxxxxxxxxxxxxxx.yyy + 1) - zzzzz = '%s-%s' % (xxxxxxxxxxxxxxxxxxxxxxx + 1, - xxxxxxxxxxxxxxxxxxxxx + 1) - zzzzz = '%s-%s'.ww(xxxxxxxxxxxxxxxxxxxxxxx + 1, - xxxxxxxxxxxxxxxxxxxxx + 1) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testAlignClosingBracketWithVisualIndentation(self): - unformatted_code = textwrap.dedent("""\ - TEST_LIST = ('foo', 'bar', # first comment - 'baz' # second comment - ) - """) - expected_formatted_code = textwrap.dedent("""\ - TEST_LIST = ('foo', - 'bar', # first comment - 'baz' # second comment - ) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - unformatted_code = textwrap.dedent("""\ - def f(): - - def g(): - while (xxxxxxxxxxxxxxxxxxxx(yyyyyyyyyyyyy[zzzzz]) == 'aaaaaaaaaaa' and - xxxxxxxxxxxxxxxxxxxx(yyyyyyyyyyyyy[zzzzz].aaaaaaaa[0]) == 'bbbbbbb' - ): - pass - """) - expected_formatted_code = textwrap.dedent("""\ - def f(): - def g(): - while (xxxxxxxxxxxxxxxxxxxx(yyyyyyyyyyyyy[zzzzz]) == 'aaaaaaaaaaa' and - xxxxxxxxxxxxxxxxxxxx(yyyyyyyyyyyyy[zzzzz].aaaaaaaa[0]) == - 'bbbbbbb'): - pass - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testIndentSizeChanging(self): - unformatted_code = textwrap.dedent("""\ - if True: - runtime_mins = (program_end_time - program_start_time).total_seconds() / 60.0 - """) - expected_formatted_code = textwrap.dedent("""\ - if True: - runtime_mins = ( - program_end_time - program_start_time).total_seconds() / 60.0 - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testHangingIndentCollision(self): - unformatted_code = textwrap.dedent("""\ - if (aaaaaaaaaaaaaa + bbbbbbbbbbbbbbbb == ccccccccccccccccc and xxxxxxxxxxxxx or yyyyyyyyyyyyyyyyy): - pass - elif (xxxxxxxxxxxxxxx(aaaaaaaaaaa, bbbbbbbbbbbbbb, cccccccccccc, dddddddddd=None)): - pass - - - def h(): - if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): - pass - - for connection in itertools.chain(branch.contact, branch.address, morestuff.andmore.andmore.andmore.andmore.andmore.andmore.andmore): - dosomething(connection) - """) - expected_formatted_code = textwrap.dedent("""\ - if (aaaaaaaaaaaaaa + bbbbbbbbbbbbbbbb == ccccccccccccccccc and xxxxxxxxxxxxx or - yyyyyyyyyyyyyyyyy): - pass - elif (xxxxxxxxxxxxxxx( - aaaaaaaaaaa, bbbbbbbbbbbbbb, cccccccccccc, dddddddddd=None)): - pass - - - def h(): - if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and - xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): - pass - - for connection in itertools.chain( - branch.contact, branch.address, - morestuff.andmore.andmore.andmore.andmore.andmore.andmore.andmore): - dosomething(connection) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testB20016122(self): - try: - style.SetGlobalStyle( - style.CreateStyleFromConfig( - '{based_on_style: pep8, SPLIT_PENALTY_IMPORT_NAMES: 35}')) - unformatted_code = textwrap.dedent("""\ - from a_very_long_or_indented_module_name_yada_yada import (long_argument_1, - long_argument_2) - """) - expected_formatted_code = textwrap.dedent("""\ - from a_very_long_or_indented_module_name_yada_yada import ( - long_argument_1, long_argument_2) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, - reformatter.Reformat(uwlines)) - finally: - style.SetGlobalStyle(style.CreatePEP8Style()) - - def testSplittingBeforeLogicalOperator(self): - try: - style.SetGlobalStyle( - style.CreateStyleFromConfig( - '{based_on_style: pep8, split_before_logical_operator: True}')) - unformatted_code = textwrap.dedent("""\ - def foo(): - return bool(update.message.new_chat_member or update.message.left_chat_member or - update.message.new_chat_title or update.message.new_chat_photo or - update.message.delete_chat_photo or update.message.group_chat_created or - update.message.supergroup_chat_created or update.message.channel_chat_created - or update.message.migrate_to_chat_id or update.message.migrate_from_chat_id or - update.message.pinned_message) - """) - expected_formatted_code = textwrap.dedent("""\ - def foo(): - return bool( - update.message.new_chat_member or update.message.left_chat_member - or update.message.new_chat_title or update.message.new_chat_photo - or update.message.delete_chat_photo - or update.message.group_chat_created - or update.message.supergroup_chat_created - or update.message.channel_chat_created - or update.message.migrate_to_chat_id - or update.message.migrate_from_chat_id - or update.message.pinned_message) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, - reformatter.Reformat(uwlines)) - finally: - style.SetGlobalStyle(style.CreatePEP8Style()) - - def testContiguousListEndingWithComment(self): - unformatted_code = textwrap.dedent("""\ - if True: - if True: - keys.append(aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa) # may be unassigned. - """) - expected_formatted_code = textwrap.dedent("""\ - if True: - if True: - keys.append( - aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa) # may be unassigned. - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testSplittingBeforeFirstArgument(self): - try: - style.SetGlobalStyle( - style.CreateStyleFromConfig( - '{based_on_style: pep8, split_before_first_argument: True}')) - unformatted_code = textwrap.dedent("""\ - a_very_long_function_name(long_argument_name_1=1, long_argument_name_2=2, - long_argument_name_3=3, long_argument_name_4=4) - """) - expected_formatted_code = textwrap.dedent("""\ - a_very_long_function_name( - long_argument_name_1=1, - long_argument_name_2=2, - long_argument_name_3=3, - long_argument_name_4=4) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, - reformatter.Reformat(uwlines)) - finally: - style.SetGlobalStyle(style.CreatePEP8Style()) - - -class TestingNotInParameters(unittest.TestCase): - - def testNotInParams(self): - unformatted_code = textwrap.dedent("""\ - list("a long line to break the line. a long line to break the brk a long lin", not True) - """) - - expected_code = textwrap.dedent("""\ - list("a long line to break the line. a long line to break the brk a long lin", - not True) - """) - - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertEqual(expected_code, reformatter.Reformat(uwlines)) - - -@unittest.skipIf(py3compat.PY3, 'Requires Python 2') -class TestVerifyNoVerify(ReformatterTest): - - @classmethod - def setUpClass(cls): - style.SetGlobalStyle(style.CreatePEP8Style()) - - def testVerifyException(self): - unformatted_code = textwrap.dedent("""\ - class ABC(metaclass=type): - pass - """) - uwlines = _ParseAndUnwrap(unformatted_code) - with self.assertRaises(verifier.InternalError): - reformatter.Reformat(uwlines, verify=True) - with self.assertRaises(verifier.InternalError): - # default should be True - reformatter.Reformat(uwlines) - - def testNoVerify(self): - unformatted_code = textwrap.dedent("""\ - class ABC(metaclass=type): - pass - """) - expected_formatted_code = textwrap.dedent("""\ - class ABC(metaclass=type): - pass - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual( - expected_formatted_code, reformatter.Reformat( - uwlines, verify=False)) - - def testVerifyFutureImport(self): - unformatted_code = textwrap.dedent("""\ - from __future__ import print_function - - def call_my_function(the_function): - the_function("hi") - - if __name__ == "__main__": - call_my_function(print) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - with self.assertRaises(verifier.InternalError): - reformatter.Reformat(uwlines, verify=True) - - expected_formatted_code = textwrap.dedent("""\ - from __future__ import print_function - - - def call_my_function(the_function): - the_function("hi") - - - if __name__ == "__main__": - call_my_function(print) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual( - expected_formatted_code, reformatter.Reformat( - uwlines, verify=False)) - - def testContinuationLineShouldBeDistinguished(self): - unformatted_code = textwrap.dedent("""\ - class Foo(object): - - def bar(self): - if self.solo_generator_that_is_long is None and len( - self.generators + self.next_batch) == 1: - pass - """) - expected_formatted_code = textwrap.dedent("""\ - class Foo(object): - def bar(self): - if self.solo_generator_that_is_long is None and len( - self.generators + self.next_batch) == 1: - pass - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual( - expected_formatted_code, reformatter.Reformat( - uwlines, verify=False)) - - -@unittest.skipUnless(py3compat.PY3, 'Requires Python 3') -class TestsForPython3Code(ReformatterTest): - """Test a few constructs that are new Python 3 syntax.""" - - @classmethod - def setUpClass(cls): - style.SetGlobalStyle(style.CreatePEP8Style()) - - def testKeywordOnlyArgSpecifier(self): - unformatted_code = textwrap.dedent("""\ - def foo(a, *, kw): - return a+kw - """) - expected_formatted_code = textwrap.dedent("""\ - def foo(a, *, kw): - return a + kw - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testAnnotations(self): - unformatted_code = textwrap.dedent("""\ - def foo(a: list, b: "bar") -> dict: - return a+b - """) - expected_formatted_code = textwrap.dedent("""\ - def foo(a: list, b: "bar") -> dict: - return a + b - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testExecAsNonKeyword(self): - unformatted_code = 'methods.exec( sys.modules[name])\n' - expected_formatted_code = 'methods.exec(sys.modules[name])\n' - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testAsyncFunctions(self): - if sys.version_info[1] < 5: - return - code = textwrap.dedent("""\ - import asyncio - import time - - - @print_args - async def slow_operation(): - await asyncio.sleep(1) - # print("Slow operation {} complete".format(n)) - - - async def main(): - start = time.time() - if (await get_html()): - pass - """) - uwlines = _ParseAndUnwrap(code) - self.assertCodeEqual(code, reformatter.Reformat(uwlines, verify=False)) - - def testNoSpacesAroundPowerOparator(self): - try: - style.SetGlobalStyle( - style.CreateStyleFromConfig( - '{based_on_style: pep8, SPACES_AROUND_POWER_OPERATOR: True}')) - unformatted_code = textwrap.dedent("""\ - a**b - """) - expected_formatted_code = textwrap.dedent("""\ - a ** b - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, - reformatter.Reformat(uwlines)) - finally: - style.SetGlobalStyle(style.CreatePEP8Style()) - - -class TestsForFBStyle(ReformatterTest): - - @classmethod - def setUpClass(cls): - style.SetGlobalStyle(style.CreateFacebookStyle()) - - def testNoNeedForLineBreaks(self): - unformatted_code = textwrap.dedent("""\ - def overly_long_function_name( - just_one_arg, **kwargs): - pass - """) - expected_formatted_code = textwrap.dedent("""\ - def overly_long_function_name(just_one_arg, **kwargs): - pass - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testDedentClosingBracket(self): - unformatted_code = textwrap.dedent("""\ - def overly_long_function_name( - first_argument_on_the_same_line, - second_argument_makes_the_line_too_long): - pass - """) - expected_formatted_code = textwrap.dedent("""\ - def overly_long_function_name( - first_argument_on_the_same_line, second_argument_makes_the_line_too_long - ): - pass - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testBreakAfterOpeningBracketIfContentsTooBig(self): - unformatted_code = textwrap.dedent("""\ - def overly_long_function_name(a, b, c, d, e, f, g, h, i, j, k, l, m, - n, o, p, q, r, s, t, u, v, w, x, y, z): - pass - """) - expected_formatted_code = textwrap.dedent("""\ - def overly_long_function_name( - a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, \ -v, w, x, y, z - ): - pass - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testDedentClosingBracketWithComments(self): - unformatted_code = textwrap.dedent("""\ - def overly_long_function_name( - # comment about the first argument - first_argument_with_a_very_long_name_or_so, - # comment about the second argument - second_argument_makes_the_line_too_long): - pass - """) - expected_formatted_code = textwrap.dedent("""\ - def overly_long_function_name( - # comment about the first argument - first_argument_with_a_very_long_name_or_so, - # comment about the second argument - second_argument_makes_the_line_too_long - ): - pass - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testDedentImportAsNames(self): - unformatted_code = textwrap.dedent("""\ - from module import ( - internal_function as function, - SOME_CONSTANT_NUMBER1, - SOME_CONSTANT_NUMBER2, - SOME_CONSTANT_NUMBER3, - ) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(unformatted_code, reformatter.Reformat(uwlines)) - - def testDedentTestListGexp(self): - # TODO(ambv): Arguably _DetermineMustSplitAnnotation shouldn't enforce - # breaks only on the basis of a trailing comma if the entire thing fits - # in a single line. - unformatted_code = textwrap.dedent("""\ - try: - pass - except ( - IOError, OSError, LookupError, RuntimeError, OverflowError - ) as exception: - pass - - try: - pass - except ( - IOError, - OSError, - LookupError, - RuntimeError, - OverflowError, - ) as exception: - pass - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(unformatted_code, reformatter.Reformat(uwlines)) - - def testBrokenIdempotency(self): - # TODO(ambv): The following behaviour should be fixed. - pass0_code = textwrap.dedent("""\ - try: - pass - except (IOError, OSError, LookupError, RuntimeError, OverflowError) as exception: - pass - """) - pass1_code = textwrap.dedent("""\ - try: - pass - except ( - IOError, OSError, LookupError, RuntimeError, OverflowError - ) as exception: - pass - """) - uwlines = _ParseAndUnwrap(pass0_code) - self.assertCodeEqual(pass1_code, reformatter.Reformat(uwlines)) - - pass2_code = textwrap.dedent("""\ - try: - pass - except ( - IOError, OSError, LookupError, RuntimeError, OverflowError - ) as exception: - pass - """) - uwlines = _ParseAndUnwrap(pass1_code) - self.assertCodeEqual(pass2_code, reformatter.Reformat(uwlines)) - - def testIfExprHangingIndent(self): - unformatted_code = textwrap.dedent("""\ - if True: - if True: - if True: - if not self.frobbies and ( - self.foobars.counters['db.cheeses'] != 1 or - self.foobars.counters['db.marshmellow_skins'] != 1): - pass - """) - expected_formatted_code = textwrap.dedent("""\ - if True: - if True: - if True: - if not self.frobbies and ( - self.foobars.counters['db.cheeses'] != 1 or - self.foobars.counters['db.marshmellow_skins'] != 1 - ): - pass - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testSimpleDedenting(self): - unformatted_code = textwrap.dedent("""\ - if True: - self.assertEqual(result.reason_not_added, "current preflight is still running") - """) - expected_formatted_code = textwrap.dedent("""\ - if True: - self.assertEqual( - result.reason_not_added, "current preflight is still running" - ) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testDedentingWithSubscripts(self): - unformatted_code = textwrap.dedent("""\ - class Foo: - class Bar: - @classmethod - def baz(cls, clues_list, effect, constraints, constraint_manager): - if clues_lists: - return cls.single_constraint_not(clues_lists, effect, constraints[0], constraint_manager) - - """) - expected_formatted_code = textwrap.dedent("""\ - class Foo: - class Bar: - @classmethod - def baz(cls, clues_list, effect, constraints, constraint_manager): - if clues_lists: - return cls.single_constraint_not( - clues_lists, effect, constraints[0], constraint_manager - ) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testDedentingCallsWithInnerLists(self): - unformatted_code = textwrap.dedent("""\ - class _(): - def _(): - cls.effect_clues = { - 'effect': Clue((cls.effect_time, 'apache_host'), effect_line, 40) - } - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(unformatted_code, reformatter.Reformat(uwlines)) - - def testDedentingListComprehension(self): - unformatted_code = textwrap.dedent("""\ - class Foo(): - def _pack_results_for_constraint_or(): - self.param_groups = dict( - ( - key + 1, ParamGroup(groups[key], default_converter) - ) for key in six.moves.range(len(groups)) - ) - - for combination in cls._clues_combinations(clues_lists): - if all( - cls._verify_constraint(combination, effect, constraint) - for constraint in constraints - ): - pass - - guessed_dict = dict( - ( - key, guessed_pattern_matches[key] - ) for key in six.moves.range(len(guessed_pattern_matches)) - ) - - content = "".join( - itertools.chain( - (first_line_fragment, ), lines_between, (last_line_fragment, ) - ) - ) - - rule = Rule( - [self.cause1, self.cause2, self.cause1, self.cause2], self.effect, constraints1, - Rule.LINKAGE_AND - ) - - assert sorted(log_type.files_to_parse) == [ - ('localhost', os.path.join(path, 'node_1.log'), super_parser), - ('localhost', os.path.join(path, 'node_2.log'), super_parser) - ] - """) - expected_formatted_code = textwrap.dedent("""\ - class Foo(): - def _pack_results_for_constraint_or(): - self.param_groups = dict( - (key + 1, ParamGroup(groups[key], default_converter)) - for key in six.moves.range(len(groups)) - ) - - for combination in cls._clues_combinations(clues_lists): - if all( - cls._verify_constraint(combination, effect, constraint) - for constraint in constraints - ): - pass - - guessed_dict = dict( - (key, guessed_pattern_matches[key]) - for key in six.moves.range(len(guessed_pattern_matches)) - ) - - content = "".join( - itertools.chain( - (first_line_fragment, ), lines_between, (last_line_fragment, ) - ) - ) - - rule = Rule( - [self.cause1, self.cause2, self.cause1, self.cause2], self.effect, - constraints1, Rule.LINKAGE_AND - ) - - assert sorted(log_type.files_to_parse) == [ - ('localhost', os.path.join(path, 'node_1.log'), super_parser), - ('localhost', os.path.join(path, 'node_2.log'), super_parser) - ] - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) - - def testMustSplitDedenting(self): - unformatted_code = textwrap.dedent("""\ - class _(): - def _(): - effect_line = FrontInput( - effect_line_offset, line_content, - LineSource('localhost', xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx) - ) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(unformatted_code, reformatter.Reformat(uwlines)) - - def testDedentIfConditional(self): - unformatted_code = textwrap.dedent("""\ - class _(): - def _(): - if True: - if not self.frobbies and ( - self.foobars.counters['db.cheeses'] != 1 or - self.foobars.counters['db.marshmellow_skins'] != 1 - ): - pass - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(unformatted_code, reformatter.Reformat(uwlines)) - - def testDedentSet(self): - unformatted_code = textwrap.dedent("""\ - class _(): - def _(): - assert set(self.constraint_links.get_links()) == set( - [ - (2, 10, 100), - (2, 10, 200), - (2, 20, 100), - (2, 20, 200), - ] - ) - """) - uwlines = _ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(unformatted_code, reformatter.Reformat(uwlines)) - - def testDedentingInnerScope(self): - code = textwrap.dedent("""\ - class Foo(): - @classmethod - def _pack_results_for_constraint_or(cls, combination, constraints): - return cls._create_investigation_result( - (clue for clue in combination if not clue == Verifier.UNMATCHED), - constraints, InvestigationResult.OR - ) - """) - uwlines = _ParseAndUnwrap(code) - reformatted_code = reformatter.Reformat(uwlines) - self.assertCodeEqual(code, reformatted_code) - - uwlines = _ParseAndUnwrap(reformatted_code) - reformatted_code = reformatter.Reformat(uwlines) - self.assertCodeEqual(code, reformatted_code) - - -def _ParseAndUnwrap(code, dumptree=False): - """Produces unwrapped lines from the given code. - - Parses the code into a tree, performs comment splicing and runs the - unwrapper. - - Arguments: - code: code to parse as a string - dumptree: if True, the parsed pytree (after comment splicing) is dumped - to stderr. Useful for debugging. - - Returns: - List of unwrapped lines. - """ - tree = pytree_utils.ParseCodeToTree(code) - comment_splicer.SpliceComments(tree) - continuation_splicer.SpliceContinuations(tree) - subtype_assigner.AssignSubtypes(tree) - split_penalty.ComputeSplitPenalties(tree) - blank_line_calculator.CalculateBlankLines(tree) - - if dumptree: - pytree_visitor.DumpPyTree(tree, target_stream=sys.stderr) - - uwlines = pytree_unwrapper.UnwrapPyTree(tree) - for uwl in uwlines: - uwl.CalculateFormattingInformation() - - return uwlines - - -if __name__ == '__main__': - unittest.main() diff --git a/yapftests/split_penalty_test.py b/yapftests/split_penalty_test.py index 51f8f3e06..dd5d0598e 100644 --- a/yapftests/split_penalty_test.py +++ b/yapftests/split_penalty_test.py @@ -1,4 +1,4 @@ -# Copyright 2015-2016 Google Inc. All Rights Reserved. +# Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,19 +17,26 @@ import textwrap import unittest -from lib2to3 import pytree +from yapf_third_party._ylib2to3 import pytree -from yapf.yapflib import pytree_utils -from yapf.yapflib import pytree_visitor -from yapf.yapflib import split_penalty +from yapf.pytree import pytree_utils +from yapf.pytree import pytree_visitor +from yapf.pytree import split_penalty +from yapf.yapflib import style + +from yapftests import yapf_test_helper UNBREAKABLE = split_penalty.UNBREAKABLE +VERY_STRONGLY_CONNECTED = split_penalty.VERY_STRONGLY_CONNECTED DOTTED_NAME = split_penalty.DOTTED_NAME STRONGLY_CONNECTED = split_penalty.STRONGLY_CONNECTED -CONTIGUOUS_LIST = split_penalty.CONTIGUOUS_LIST -class SplitPenaltyTest(unittest.TestCase): +class SplitPenaltyTest(yapf_test_helper.YAPFTest): + + @classmethod + def setUpClass(cls): + style.SetGlobalStyle(style.CreateYapfStyle()) def _ParseAndComputePenalties(self, code, dumptree=False): """Parses the code and computes split penalties. @@ -61,8 +68,9 @@ def FlattenRec(tree): if pytree_utils.NodeName(tree) in pytree_utils.NONSEMANTIC_TOKENS: return [] if isinstance(tree, pytree.Leaf): - return [(tree.value, pytree_utils.GetNodeAnnotation( - tree, pytree_utils.Annotation.SPLIT_PENALTY))] + return [(tree.value, + pytree_utils.GetNodeAnnotation( + tree, pytree_utils.Annotation.SPLIT_PENALTY))] nodes = [] for node in tree.children: nodes += FlattenRec(node) @@ -72,10 +80,10 @@ def FlattenRec(tree): def testUnbreakable(self): # Test function definitions. - code = textwrap.dedent(r""" - def foo(x): - pass - """) + code = textwrap.dedent("""\ + def foo(x): + pass + """) tree = self._ParseAndComputePenalties(code) self._CheckPenalties(tree, [ ('def', None), @@ -85,13 +93,13 @@ def foo(x): (')', STRONGLY_CONNECTED), (':', UNBREAKABLE), ('pass', None), - ]) # yapf: disable + ]) # Test function definition with trailing comment. - code = textwrap.dedent(r""" - def foo(x): # trailing comment - pass - """) + code = textwrap.dedent("""\ + def foo(x): # trailing comment + pass + """) tree = self._ParseAndComputePenalties(code) self._CheckPenalties(tree, [ ('def', None), @@ -101,15 +109,15 @@ def foo(x): # trailing comment (')', STRONGLY_CONNECTED), (':', UNBREAKABLE), ('pass', None), - ]) # yapf: disable + ]) # Test class definitions. - code = textwrap.dedent(r""" - class A: - pass - class B(A): - pass - """) + code = textwrap.dedent("""\ + class A: + pass + class B(A): + pass + """) tree = self._ParseAndComputePenalties(code) self._CheckPenalties(tree, [ ('class', None), @@ -123,26 +131,26 @@ class B(A): (')', None), (':', UNBREAKABLE), ('pass', None), - ]) # yapf: disable + ]) # Test lambda definitions. - code = textwrap.dedent(r""" - lambda a, b: None - """) + code = textwrap.dedent("""\ + lambda a, b: None + """) tree = self._ParseAndComputePenalties(code) self._CheckPenalties(tree, [ ('lambda', None), - ('a', UNBREAKABLE), - (',', UNBREAKABLE), - ('b', UNBREAKABLE), - (':', UNBREAKABLE), - ('None', UNBREAKABLE), - ]) # yapf: disable + ('a', VERY_STRONGLY_CONNECTED), + (',', VERY_STRONGLY_CONNECTED), + ('b', VERY_STRONGLY_CONNECTED), + (':', VERY_STRONGLY_CONNECTED), + ('None', VERY_STRONGLY_CONNECTED), + ]) # Test dotted names. - code = textwrap.dedent(r""" - import a.b.c - """) + code = textwrap.dedent("""\ + import a.b.c + """) tree = self._ParseAndComputePenalties(code) self._CheckPenalties(tree, [ ('import', None), @@ -151,19 +159,21 @@ class B(A): ('b', UNBREAKABLE), ('.', UNBREAKABLE), ('c', UNBREAKABLE), - ]) # yapf: disable + ]) def testStronglyConnected(self): # Test dictionary keys. - code = textwrap.dedent(r""" - a = { - 'x': 42, - y(lambda a: 23): 37, - } - """) + code = textwrap.dedent("""\ + a = { + 'x': 42, + y(lambda a: 23): 37, + } + """) tree = self._ParseAndComputePenalties(code) self._CheckPenalties(tree, [ - ('a', None), ('=', None), ('{', None), + ('a', None), + ('=', None), + ('{', None), ("'x'", None), (':', STRONGLY_CONNECTED), ('42', None), @@ -171,66 +181,91 @@ def testStronglyConnected(self): ('y', None), ('(', UNBREAKABLE), ('lambda', STRONGLY_CONNECTED), - ('a', UNBREAKABLE), - (':', UNBREAKABLE), - ('23', UNBREAKABLE), - (')', UNBREAKABLE), + ('a', VERY_STRONGLY_CONNECTED), + (':', VERY_STRONGLY_CONNECTED), + ('23', VERY_STRONGLY_CONNECTED), + (')', VERY_STRONGLY_CONNECTED), (':', STRONGLY_CONNECTED), ('37', None), (',', None), ('}', None), - ]) # yapf: disable + ]) # Test list comprehension. - code = textwrap.dedent(r""" - [a for a in foo if a.x == 37] - """) + code = textwrap.dedent("""\ + [a for a in foo if a.x == 37] + """) tree = self._ParseAndComputePenalties(code) self._CheckPenalties(tree, [ ('[', None), - ('a', STRONGLY_CONNECTED), - ('for', STRONGLY_CONNECTED), + ('a', None), + ('for', 0), ('a', STRONGLY_CONNECTED), ('in', STRONGLY_CONNECTED), ('foo', STRONGLY_CONNECTED), - ('if', STRONGLY_CONNECTED), + ('if', 0), ('a', STRONGLY_CONNECTED), - ('.', UNBREAKABLE), + ('.', VERY_STRONGLY_CONNECTED), ('x', DOTTED_NAME), ('==', STRONGLY_CONNECTED), ('37', STRONGLY_CONNECTED), - (']', STRONGLY_CONNECTED), - ]) # yapf: disable + (']', None), + ]) def testFuncCalls(self): - code = 'foo(1, 2, 3)\n' + code = textwrap.dedent("""\ + foo(1, 2, 3) + """) tree = self._ParseAndComputePenalties(code) self._CheckPenalties(tree, [ ('foo', None), ('(', UNBREAKABLE), - ('1', CONTIGUOUS_LIST), - (',', CONTIGUOUS_LIST), - ('2', CONTIGUOUS_LIST), - (',', CONTIGUOUS_LIST), - ('3', CONTIGUOUS_LIST), - (')', UNBREAKABLE)]) # yapf: disable + ('1', None), + (',', UNBREAKABLE), + ('2', None), + (',', UNBREAKABLE), + ('3', None), + (')', VERY_STRONGLY_CONNECTED), + ]) # Now a method call, which has more than one trailer - code = 'foo.bar.baz(1, 2, 3)\n' + code = textwrap.dedent("""\ + foo.bar.baz(1, 2, 3) + """) tree = self._ParseAndComputePenalties(code) self._CheckPenalties(tree, [ ('foo', None), - ('.', UNBREAKABLE), + ('.', VERY_STRONGLY_CONNECTED), ('bar', DOTTED_NAME), - ('.', STRONGLY_CONNECTED), + ('.', VERY_STRONGLY_CONNECTED), ('baz', DOTTED_NAME), ('(', STRONGLY_CONNECTED), - ('1', CONTIGUOUS_LIST), - (',', CONTIGUOUS_LIST), - ('2', CONTIGUOUS_LIST), - (',', CONTIGUOUS_LIST), - ('3', CONTIGUOUS_LIST), - (')', UNBREAKABLE)]) # yapf: disable + ('1', None), + (',', UNBREAKABLE), + ('2', None), + (',', UNBREAKABLE), + ('3', None), + (')', VERY_STRONGLY_CONNECTED), + ]) + + # Test single generator argument. + code = textwrap.dedent("""\ + max(i for i in xrange(10)) + """) + tree = self._ParseAndComputePenalties(code) + self._CheckPenalties(tree, [ + ('max', None), + ('(', UNBREAKABLE), + ('i', 0), + ('for', 0), + ('i', STRONGLY_CONNECTED), + ('in', STRONGLY_CONNECTED), + ('xrange', STRONGLY_CONNECTED), + ('(', UNBREAKABLE), + ('10', STRONGLY_CONNECTED), + (')', VERY_STRONGLY_CONNECTED), + (')', VERY_STRONGLY_CONNECTED), + ]) if __name__ == '__main__': diff --git a/yapftests/style_test.py b/yapftests/style_test.py index c8ba03bc7..64e64a5be 100644 --- a/yapftests/style_test.py +++ b/yapftests/style_test.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2015-2016 Google Inc. All Rights Reserved. +# Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ # limitations under the License. """Tests for yapf.style.""" -import contextlib +import os import shutil import tempfile import textwrap @@ -22,8 +22,36 @@ from yapf.yapflib import style - -class UtilsTest(unittest.TestCase): +from yapftests import utils +from yapftests import yapf_test_helper + + +class UtilsTest(yapf_test_helper.YAPFTest): + + def testContinuationAlignStyleStringConverter(self): + for cont_align_space in ('', 'space', '"space"', '\'space\''): + self.assertEqual( + style._ContinuationAlignStyleStringConverter(cont_align_space), + 'SPACE') + for cont_align_fixed in ('fixed', '"fixed"', '\'fixed\''): + self.assertEqual( + style._ContinuationAlignStyleStringConverter(cont_align_fixed), + 'FIXED') + for cont_align_valignright in ( + 'valign-right', + '"valign-right"', + '\'valign-right\'', + 'valign_right', + '"valign_right"', + '\'valign_right\'', + ): + self.assertEqual( + style._ContinuationAlignStyleStringConverter(cont_align_valignright), + 'VALIGN-RIGHT') + with self.assertRaises(ValueError) as ctx: + style._ContinuationAlignStyleStringConverter('blahblah') + self.assertIn("unknown continuation align style: 'blahblah'", + str(ctx.exception)) def testStringListConverter(self): self.assertEqual(style._StringListConverter('foo, bar'), ['foo', 'bar']) @@ -38,42 +66,57 @@ def testBoolConverter(self): self.assertEqual(style._BoolConverter('false'), False) self.assertEqual(style._BoolConverter('0'), False) + def testIntListConverter(self): + self.assertEqual(style._IntListConverter('1, 2, 3'), [1, 2, 3]) + self.assertEqual(style._IntListConverter('[ 1, 2, 3 ]'), [1, 2, 3]) + self.assertEqual(style._IntListConverter('[ 1, 2, 3, ]'), [1, 2, 3]) -def _LooksLikeChromiumStyle(cfg): - return (cfg['INDENT_WIDTH'] == 2 and - cfg['BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF']) + def testIntOrIntListConverter(self): + self.assertEqual(style._IntOrIntListConverter('10'), 10) + self.assertEqual(style._IntOrIntListConverter('1, 2, 3'), [1, 2, 3]) def _LooksLikeGoogleStyle(cfg): - return (cfg['INDENT_WIDTH'] == 4 and - cfg['BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF']) + return cfg['COLUMN_LIMIT'] == 80 and cfg['SPLIT_COMPLEX_COMPREHENSION'] def _LooksLikePEP8Style(cfg): - return (cfg['INDENT_WIDTH'] == 4 and - not cfg['BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF']) + return cfg['COLUMN_LIMIT'] == 79 def _LooksLikeFacebookStyle(cfg): - return (cfg['INDENT_WIDTH'] == 4 and cfg['DEDENT_CLOSING_BRACKETS']) + return cfg['DEDENT_CLOSING_BRACKETS'] + + +def _LooksLikeYapfStyle(cfg): + return cfg['SPLIT_BEFORE_DOT'] -class PredefinedStylesByNameTest(unittest.TestCase): +class PredefinedStylesByNameTest(yapf_test_helper.YAPFTest): + + @classmethod + def setUpClass(cls): # pylint: disable=g-missing-super-call + style.SetGlobalStyle(style.CreatePEP8Style()) def testDefault(self): # default is PEP8 cfg = style.CreateStyleFromConfig(None) self.assertTrue(_LooksLikePEP8Style(cfg)) + def testPEP8ByName(self): + for pep8_name in ('PEP8', 'pep8', 'Pep8'): + cfg = style.CreateStyleFromConfig(pep8_name) + self.assertTrue(_LooksLikePEP8Style(cfg)) + def testGoogleByName(self): for google_name in ('google', 'Google', 'GOOGLE'): cfg = style.CreateStyleFromConfig(google_name) self.assertTrue(_LooksLikeGoogleStyle(cfg)) - def testPEP8ByName(self): - for pep8_name in ('PEP8', 'pep8', 'Pep8'): - cfg = style.CreateStyleFromConfig(pep8_name) - self.assertTrue(_LooksLikePEP8Style(cfg)) + def testYapfByName(self): + for yapf_name in ('yapf', 'YAPF'): + cfg = style.CreateStyleFromConfig(yapf_name) + self.assertTrue(_LooksLikeYapfStyle(cfg)) def testFacebookByName(self): for fb_name in ('facebook', 'FACEBOOK', 'Facebook'): @@ -81,159 +124,197 @@ def testFacebookByName(self): self.assertTrue(_LooksLikeFacebookStyle(cfg)) -@contextlib.contextmanager -def _TempFileContents(dirname, contents): - with tempfile.NamedTemporaryFile(dir=dirname, mode='w') as f: - f.write(contents) - f.flush() - yield f - - -class StyleFromFileTest(unittest.TestCase): +class StyleFromFileTest(yapf_test_helper.YAPFTest): @classmethod - def setUpClass(cls): + def setUpClass(cls): # pylint: disable=g-missing-super-call cls.test_tmpdir = tempfile.mkdtemp() + style.SetGlobalStyle(style.CreatePEP8Style()) @classmethod - def tearDownClass(cls): + def tearDownClass(cls): # pylint: disable=g-missing-super-call shutil.rmtree(cls.test_tmpdir) def testDefaultBasedOnStyle(self): - cfg = textwrap.dedent('''\ + cfg = textwrap.dedent("""\ [style] continuation_indent_width = 20 - ''') - with _TempFileContents(self.test_tmpdir, cfg) as f: - cfg = style.CreateStyleFromConfig(f.name) + """) + with utils.TempFileContents(self.test_tmpdir, cfg) as filepath: + cfg = style.CreateStyleFromConfig(filepath) self.assertTrue(_LooksLikePEP8Style(cfg)) self.assertEqual(cfg['CONTINUATION_INDENT_WIDTH'], 20) def testDefaultBasedOnPEP8Style(self): - cfg = textwrap.dedent('''\ + cfg = textwrap.dedent("""\ [style] based_on_style = pep8 continuation_indent_width = 40 - ''') - with _TempFileContents(self.test_tmpdir, cfg) as f: - cfg = style.CreateStyleFromConfig(f.name) + """) + with utils.TempFileContents(self.test_tmpdir, cfg) as filepath: + cfg = style.CreateStyleFromConfig(filepath) self.assertTrue(_LooksLikePEP8Style(cfg)) self.assertEqual(cfg['CONTINUATION_INDENT_WIDTH'], 40) - def testDefaultBasedOnChromiumStyle(self): - cfg = textwrap.dedent('''\ - [style] - based_on_style = chromium - continuation_indent_width = 30 - ''') - with _TempFileContents(self.test_tmpdir, cfg) as f: - cfg = style.CreateStyleFromConfig(f.name) - self.assertTrue(_LooksLikeChromiumStyle(cfg)) - self.assertEqual(cfg['CONTINUATION_INDENT_WIDTH'], 30) - def testDefaultBasedOnGoogleStyle(self): - cfg = textwrap.dedent('''\ + cfg = textwrap.dedent("""\ [style] based_on_style = google continuation_indent_width = 20 - ''') - with _TempFileContents(self.test_tmpdir, cfg) as f: - cfg = style.CreateStyleFromConfig(f.name) + """) + with utils.TempFileContents(self.test_tmpdir, cfg) as filepath: + cfg = style.CreateStyleFromConfig(filepath) self.assertTrue(_LooksLikeGoogleStyle(cfg)) self.assertEqual(cfg['CONTINUATION_INDENT_WIDTH'], 20) def testDefaultBasedOnFacebookStyle(self): - cfg = textwrap.dedent('''\ + cfg = textwrap.dedent("""\ [style] based_on_style = facebook continuation_indent_width = 20 - ''') - with _TempFileContents(self.test_tmpdir, cfg) as f: - cfg = style.CreateStyleFromConfig(f.name) + """) + with utils.TempFileContents(self.test_tmpdir, cfg) as filepath: + cfg = style.CreateStyleFromConfig(filepath) self.assertTrue(_LooksLikeFacebookStyle(cfg)) self.assertEqual(cfg['CONTINUATION_INDENT_WIDTH'], 20) def testBoolOptionValue(self): - cfg = textwrap.dedent('''\ + cfg = textwrap.dedent("""\ [style] - based_on_style = chromium + based_on_style = pep8 SPLIT_BEFORE_NAMED_ASSIGNS=False split_before_logical_operator = true - ''') - with _TempFileContents(self.test_tmpdir, cfg) as f: - cfg = style.CreateStyleFromConfig(f.name) - self.assertTrue(_LooksLikeChromiumStyle(cfg)) + """) + with utils.TempFileContents(self.test_tmpdir, cfg) as filepath: + cfg = style.CreateStyleFromConfig(filepath) + self.assertTrue(_LooksLikePEP8Style(cfg)) self.assertEqual(cfg['SPLIT_BEFORE_NAMED_ASSIGNS'], False) self.assertEqual(cfg['SPLIT_BEFORE_LOGICAL_OPERATOR'], True) def testStringListOptionValue(self): - cfg = textwrap.dedent('''\ + cfg = textwrap.dedent("""\ [style] - based_on_style = chromium + based_on_style = pep8 I18N_FUNCTION_CALL = N_, V_, T_ - ''') - with _TempFileContents(self.test_tmpdir, cfg) as f: - cfg = style.CreateStyleFromConfig(f.name) - self.assertTrue(_LooksLikeChromiumStyle(cfg)) + """) + with utils.TempFileContents(self.test_tmpdir, cfg) as filepath: + cfg = style.CreateStyleFromConfig(filepath) + self.assertTrue(_LooksLikePEP8Style(cfg)) self.assertEqual(cfg['I18N_FUNCTION_CALL'], ['N_', 'V_', 'T_']) def testErrorNoStyleFile(self): - with self.assertRaisesRegexp(style.StyleConfigError, - 'is not a valid style or file path'): + with self.assertRaisesRegex(style.StyleConfigError, + 'is not a valid style or file path'): style.CreateStyleFromConfig('/8822/xyznosuchfile') def testErrorNoStyleSection(self): - cfg = textwrap.dedent('''\ + cfg = textwrap.dedent("""\ [s] indent_width=2 - ''') - with _TempFileContents(self.test_tmpdir, cfg) as f: - with self.assertRaisesRegexp(style.StyleConfigError, - 'Unable to find section'): - style.CreateStyleFromConfig(f.name) + """) + with utils.TempFileContents(self.test_tmpdir, cfg) as filepath: + with self.assertRaisesRegex(style.StyleConfigError, + 'Unable to find section'): + style.CreateStyleFromConfig(filepath) def testErrorUnknownStyleOption(self): - cfg = textwrap.dedent('''\ + cfg = textwrap.dedent("""\ [style] indent_width=2 hummus=2 - ''') - with _TempFileContents(self.test_tmpdir, cfg) as f: - with self.assertRaisesRegexp(style.StyleConfigError, - 'Unknown style option'): - style.CreateStyleFromConfig(f.name) + """) + with utils.TempFileContents(self.test_tmpdir, cfg) as filepath: + with self.assertRaisesRegex(style.StyleConfigError, + 'Unknown style option'): + style.CreateStyleFromConfig(filepath) + + def testPyprojectTomlNoYapfSection(self): + filepath = os.path.join(self.test_tmpdir, 'pyproject.toml') + _ = open(filepath, 'w') + with self.assertRaisesRegex(style.StyleConfigError, + 'Unable to find section'): + style.CreateStyleFromConfig(filepath) + + def testPyprojectTomlParseYapfSection(self): + + cfg = textwrap.dedent("""\ + [tool.yapf] + based_on_style = "pep8" + continuation_indent_width = 40 + """) + filepath = os.path.join(self.test_tmpdir, 'pyproject.toml') + with open(filepath, 'w') as f: + f.write(cfg) + cfg = style.CreateStyleFromConfig(filepath) + self.assertTrue(_LooksLikePEP8Style(cfg)) + self.assertEqual(cfg['CONTINUATION_INDENT_WIDTH'], 40) + +class StyleFromDict(yapf_test_helper.YAPFTest): -class StyleFromCommandLine(unittest.TestCase): + @classmethod + def setUpClass(cls): # pylint: disable=g-missing-super-call + style.SetGlobalStyle(style.CreatePEP8Style()) + + def testDefaultBasedOnStyle(self): + config_dict = { + 'based_on_style': 'pep8', + 'indent_width': 2, + 'blank_line_before_nested_class_or_def': True + } + cfg = style.CreateStyleFromConfig(config_dict) + self.assertTrue(_LooksLikePEP8Style(cfg)) + self.assertEqual(cfg['INDENT_WIDTH'], 2) + + def testDefaultBasedOnStyleBadDict(self): + self.assertRaisesRegex(style.StyleConfigError, 'Unknown style option', + style.CreateStyleFromConfig, + {'based_on_styl': 'pep8'}) + self.assertRaisesRegex(style.StyleConfigError, 'not a valid', + style.CreateStyleFromConfig, + {'INDENT_WIDTH': 'FOUR'}) + + +class StyleFromCommandLine(yapf_test_helper.YAPFTest): + + @classmethod + def setUpClass(cls): # pylint: disable=g-missing-super-call + style.SetGlobalStyle(style.CreatePEP8Style()) def testDefaultBasedOnStyle(self): cfg = style.CreateStyleFromConfig( '{based_on_style: pep8,' ' indent_width: 2,' ' blank_line_before_nested_class_or_def: True}') - self.assertTrue(_LooksLikeChromiumStyle(cfg)) + self.assertTrue(_LooksLikePEP8Style(cfg)) self.assertEqual(cfg['INDENT_WIDTH'], 2) def testDefaultBasedOnStyleNotStrict(self): cfg = style.CreateStyleFromConfig( - '{based_on_style : pep8' - ' ,indent_width=2' + '{based_on_style : pep8,' + ' indent_width=2' ' blank_line_before_nested_class_or_def:True}') - self.assertTrue(_LooksLikeChromiumStyle(cfg)) + self.assertTrue(_LooksLikePEP8Style(cfg)) self.assertEqual(cfg['INDENT_WIDTH'], 2) + def testDefaultBasedOnExplicitlyUnicodeTypeString(self): + cfg = style.CreateStyleFromConfig('{}') + self.assertIsInstance(cfg, dict) + + def testDefaultBasedOnDetaultTypeString(self): + cfg = style.CreateStyleFromConfig('{}') + self.assertIsInstance(cfg, dict) + def testDefaultBasedOnStyleBadString(self): - self.assertRaisesRegexp(style.StyleConfigError, 'Unknown style option', - style.CreateStyleFromConfig, - '{based_on_styl: pep8}') - self.assertRaisesRegexp(style.StyleConfigError, 'not a valid', - style.CreateStyleFromConfig, '{INDENT_WIDTH: FOUR}') - self.assertRaisesRegexp(style.StyleConfigError, 'Invalid style dict', - style.CreateStyleFromConfig, - '{based_on_style: pep8') + self.assertRaisesRegex(style.StyleConfigError, 'Unknown style option', + style.CreateStyleFromConfig, '{based_on_styl: pep8}') + self.assertRaisesRegex(style.StyleConfigError, 'not a valid', + style.CreateStyleFromConfig, '{INDENT_WIDTH: FOUR}') + self.assertRaisesRegex(style.StyleConfigError, 'Invalid style dict', + style.CreateStyleFromConfig, '{based_on_style: pep8') -class StyleHelp(unittest.TestCase): +class StyleHelp(yapf_test_helper.YAPFTest): def testHelpKeys(self): settings = sorted(style.Help()) diff --git a/yapftests/subtype_assigner_test.py b/yapftests/subtype_assigner_test.py index 9fe5cf8e1..01b371051 100644 --- a/yapftests/subtype_assigner_test.py +++ b/yapftests/subtype_assigner_test.py @@ -1,4 +1,4 @@ -# Copyright 2015-2016 Google Inc. All Rights Reserved. +# Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,51 +13,30 @@ # limitations under the License. """Tests for yapf.subtype_assigner.""" -import sys import textwrap import unittest +from yapf.pytree import pytree_utils from yapf.yapflib import format_token -from yapf.yapflib import pytree_unwrapper -from yapf.yapflib import pytree_utils -from yapf.yapflib import pytree_visitor -from yapf.yapflib import subtype_assigner +from yapf.yapflib import subtypes +from yapftests import yapf_test_helper -class SubtypeAssignerTest(unittest.TestCase): - def _ParseAndUnwrap(self, code, dumptree=False): - """Produces unwrapped lines from the given code. +class SubtypeAssignerTest(yapf_test_helper.YAPFTest): - Parses the code into a tree, assigns subtypes and runs the unwrapper. - - Arguments: - code: code to parse as a string - dumptree: if True, the parsed pytree (after comment splicing) is dumped - to stderr. Useful for debugging. - - Returns: - List of unwrapped lines. - """ - tree = pytree_utils.ParseCodeToTree(code) - subtype_assigner.AssignSubtypes(tree) - - if dumptree: - pytree_visitor.DumpPyTree(tree, target_stream=sys.stderr) - - return pytree_unwrapper.UnwrapPyTree(tree) - - def _CheckFormatTokenSubtypes(self, uwlines, list_of_expected): - """Check that the tokens in the UnwrappedLines have the expected subtypes. + def _CheckFormatTokenSubtypes(self, llines, list_of_expected): + """Check that the tokens in the LogicalLines have the expected subtypes. Args: - uwlines: list of UnwrappedLine. + llines: list of LogicalLine. list_of_expected: list of (name, subtype) pairs. Non-semantic tokens are filtered out from the expected values. """ actual = [] - for uwl in uwlines: - filtered_values = [(ft.value, ft.subtypes) for ft in uwl.tokens + for lline in llines: + filtered_values = [(ft.value, ft.subtypes) + for ft in lline.tokens if ft.name not in pytree_utils.NONSEMANTIC_TOKENS] if filtered_values: actual.append(filtered_values) @@ -65,154 +44,482 @@ def _CheckFormatTokenSubtypes(self, uwlines, list_of_expected): self.assertEqual(list_of_expected, actual) def testFuncDefDefaultAssign(self): - code = textwrap.dedent(r""" + self.maxDiff = None # pylint: disable=invalid-name + code = textwrap.dedent("""\ def foo(a=37, *b, **c): return -x[:42] - """) - uwlines = self._ParseAndUnwrap(code) - self._CheckFormatTokenSubtypes(uwlines, [ - [('def', [format_token.Subtype.NONE]), - ('foo', {format_token.Subtype.FUNC_DEF}), - ('(', [format_token.Subtype.NONE]), - ('a', {format_token.Subtype.NONE, - format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST}), - ('=', {format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN, - format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST}), - ('37', {format_token.Subtype.NONE, - format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST}), - (',', {format_token.Subtype.NONE}), - ('*', {format_token.Subtype.VARARGS_STAR, - format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST}), - ('b', {format_token.Subtype.NONE, - format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST}), - (',', {format_token.Subtype.NONE}), - ('**', {format_token.Subtype.KWARGS_STAR_STAR, - format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST}), - ('c', {format_token.Subtype.NONE, - format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST}), - (')', [format_token.Subtype.NONE]), - (':', [format_token.Subtype.NONE])], - [('return', [format_token.Subtype.NONE]), - ('-', {format_token.Subtype.UNARY_OPERATOR}), - ('x', [format_token.Subtype.NONE]), - ('[', {format_token.Subtype.SUBSCRIPT_BRACKET}), - (':', {format_token.Subtype.SUBSCRIPT_COLON}), - ('42', [format_token.Subtype.NONE]), - (']', {format_token.Subtype.SUBSCRIPT_BRACKET})] - ]) # yapf: disable + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckFormatTokenSubtypes(llines, [ + [ + ('def', {subtypes.NONE}), + ('foo', {subtypes.FUNC_DEF}), + ('(', {subtypes.NONE}), + ('a', { + subtypes.NONE, + subtypes.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST, + subtypes.PARAMETER_START, + }), + ('=', { + subtypes.DEFAULT_OR_NAMED_ASSIGN, + subtypes.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST, + }), + ('37', { + subtypes.NONE, + subtypes.PARAMETER_STOP, + subtypes.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST, + }), + (',', {subtypes.NONE}), + ('*', { + subtypes.PARAMETER_START, + subtypes.VARARGS_STAR, + subtypes.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST, + }), + ('b', { + subtypes.NONE, + subtypes.PARAMETER_STOP, + subtypes.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST, + }), + (',', {subtypes.NONE}), + ('**', { + subtypes.PARAMETER_START, + subtypes.KWARGS_STAR_STAR, + subtypes.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST, + }), + ('c', { + subtypes.NONE, + subtypes.PARAMETER_STOP, + subtypes.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST, + }), + (')', {subtypes.NONE}), + (':', {subtypes.NONE}), + ], + [ + ('return', {subtypes.NONE}), + ('-', {subtypes.UNARY_OPERATOR}), + ('x', {subtypes.NONE}), + ('[', {subtypes.SUBSCRIPT_BRACKET}), + (':', {subtypes.SUBSCRIPT_COLON}), + ('42', {subtypes.NONE}), + (']', {subtypes.SUBSCRIPT_BRACKET}), + ], + ]) def testFuncCallWithDefaultAssign(self): - code = textwrap.dedent(r""" + code = textwrap.dedent("""\ foo(x, a='hello world') - """) - uwlines = self._ParseAndUnwrap(code) - self._CheckFormatTokenSubtypes(uwlines, [ - [('foo', [format_token.Subtype.NONE]), - ('(', [format_token.Subtype.NONE]), - ('x', {format_token.Subtype.NONE, - format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST}), - (',', {format_token.Subtype.NONE}), - ('a', {format_token.Subtype.NONE, - format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST}), - ('=', {format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN}), - ("'hello world'", {format_token.Subtype.NONE}), - (')', [format_token.Subtype.NONE])] - ]) # yapf: disable + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckFormatTokenSubtypes(llines, [ + [ + ('foo', {subtypes.NONE}), + ('(', {subtypes.NONE}), + ('x', { + subtypes.NONE, + subtypes.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST, + }), + (',', {subtypes.NONE}), + ('a', { + subtypes.NONE, + subtypes.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST, + }), + ('=', {subtypes.DEFAULT_OR_NAMED_ASSIGN}), + ("'hello world'", {subtypes.NONE}), + (')', {subtypes.NONE}), + ], + ]) def testSetComprehension(self): + code = textwrap.dedent("""\ + def foo(value): + return {value.lower()} + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckFormatTokenSubtypes(llines, [ + [ + ('def', {subtypes.NONE}), + ('foo', {subtypes.FUNC_DEF}), + ('(', {subtypes.NONE}), + ('value', { + subtypes.NONE, + subtypes.PARAMETER_START, + subtypes.PARAMETER_STOP, + }), + (')', {subtypes.NONE}), + (':', {subtypes.NONE}), + ], + [ + ('return', {subtypes.NONE}), + ('{', {subtypes.NONE}), + ('value', {subtypes.NONE}), + ('.', {subtypes.NONE}), + ('lower', {subtypes.NONE}), + ('(', {subtypes.NONE}), + (')', {subtypes.NONE}), + ('}', {subtypes.NONE}), + ], + ]) + code = textwrap.dedent("""\ def foo(strs): return {s.lower() for s in strs} - """) - uwlines = self._ParseAndUnwrap(code) - self._CheckFormatTokenSubtypes(uwlines, [ - [('def', [format_token.Subtype.NONE]), - ('foo', {format_token.Subtype.FUNC_DEF}), - ('(', [format_token.Subtype.NONE]), - ('strs', [format_token.Subtype.NONE]), - (')', [format_token.Subtype.NONE]), - (':', [format_token.Subtype.NONE])], - [('return', [format_token.Subtype.NONE]), - ('{', [format_token.Subtype.NONE]), - ('s', [format_token.Subtype.NONE]), - ('.', [format_token.Subtype.NONE]), - ('lower', [format_token.Subtype.NONE]), - ('(', [format_token.Subtype.NONE]), - (')', [format_token.Subtype.NONE]), - ('for', {format_token.Subtype.DICT_SET_GENERATOR, - format_token.Subtype.COMP_FOR}), - ('s', {format_token.Subtype.COMP_FOR}), - ('in', {format_token.Subtype.COMP_FOR}), - ('strs', {format_token.Subtype.COMP_FOR}), - ('}', [format_token.Subtype.NONE])] - ]) # yapf: disable + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckFormatTokenSubtypes(llines, [ + [ + ('def', {subtypes.NONE}), + ('foo', {subtypes.FUNC_DEF}), + ('(', {subtypes.NONE}), + ('strs', { + subtypes.NONE, + subtypes.PARAMETER_START, + subtypes.PARAMETER_STOP, + }), + (')', {subtypes.NONE}), + (':', {subtypes.NONE}), + ], + [ + ('return', {subtypes.NONE}), + ('{', {subtypes.NONE}), + ('s', {subtypes.COMP_EXPR}), + ('.', {subtypes.COMP_EXPR}), + ('lower', {subtypes.COMP_EXPR}), + ('(', {subtypes.COMP_EXPR}), + (')', {subtypes.COMP_EXPR}), + ('for', { + subtypes.DICT_SET_GENERATOR, + subtypes.COMP_FOR, + }), + ('s', {subtypes.COMP_FOR}), + ('in', {subtypes.COMP_FOR}), + ('strs', {subtypes.COMP_FOR}), + ('}', {subtypes.NONE}), + ], + ]) + + code = textwrap.dedent("""\ + def foo(strs): + return {s + s.lower() for s in strs} + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckFormatTokenSubtypes(llines, [ + [ + ('def', {subtypes.NONE}), + ('foo', {subtypes.FUNC_DEF}), + ('(', {subtypes.NONE}), + ('strs', { + subtypes.NONE, + subtypes.PARAMETER_START, + subtypes.PARAMETER_STOP, + }), + (')', {subtypes.NONE}), + (':', {subtypes.NONE}), + ], + [ + ('return', {subtypes.NONE}), + ('{', {subtypes.NONE}), + ('s', {subtypes.COMP_EXPR}), + ('+', {subtypes.BINARY_OPERATOR, subtypes.COMP_EXPR}), + ('s', {subtypes.COMP_EXPR}), + ('.', {subtypes.COMP_EXPR}), + ('lower', {subtypes.COMP_EXPR}), + ('(', {subtypes.COMP_EXPR}), + (')', {subtypes.COMP_EXPR}), + ('for', { + subtypes.DICT_SET_GENERATOR, + subtypes.COMP_FOR, + }), + ('s', {subtypes.COMP_FOR}), + ('in', {subtypes.COMP_FOR}), + ('strs', {subtypes.COMP_FOR}), + ('}', {subtypes.NONE}), + ], + ]) + + code = textwrap.dedent("""\ + def foo(strs): + return {c.lower() for s in strs for c in s} + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckFormatTokenSubtypes(llines, [ + [ + ('def', {subtypes.NONE}), + ('foo', {subtypes.FUNC_DEF}), + ('(', {subtypes.NONE}), + ('strs', { + subtypes.NONE, + subtypes.PARAMETER_START, + subtypes.PARAMETER_STOP, + }), + (')', {subtypes.NONE}), + (':', {subtypes.NONE}), + ], + [ + ('return', {subtypes.NONE}), + ('{', {subtypes.NONE}), + ('c', {subtypes.COMP_EXPR}), + ('.', {subtypes.COMP_EXPR}), + ('lower', {subtypes.COMP_EXPR}), + ('(', {subtypes.COMP_EXPR}), + (')', {subtypes.COMP_EXPR}), + ('for', { + subtypes.DICT_SET_GENERATOR, + subtypes.COMP_FOR, + subtypes.COMP_EXPR, + }), + ('s', {subtypes.COMP_FOR, subtypes.COMP_EXPR}), + ('in', {subtypes.COMP_FOR, subtypes.COMP_EXPR}), + ('strs', {subtypes.COMP_FOR, subtypes.COMP_EXPR}), + ('for', { + subtypes.DICT_SET_GENERATOR, + subtypes.COMP_FOR, + }), + ('c', {subtypes.COMP_FOR}), + ('in', {subtypes.COMP_FOR}), + ('s', {subtypes.COMP_FOR}), + ('}', {subtypes.NONE}), + ], + ]) + + def testDictComprehension(self): + code = textwrap.dedent("""\ + def foo(value): + return {value: value.lower()} + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckFormatTokenSubtypes(llines, [ + [ + ('def', {subtypes.NONE}), + ('foo', {subtypes.FUNC_DEF}), + ('(', {subtypes.NONE}), + ('value', { + subtypes.NONE, + subtypes.PARAMETER_START, + subtypes.PARAMETER_STOP, + }), + (')', {subtypes.NONE}), + (':', {subtypes.NONE}), + ], + [ + ('return', {subtypes.NONE}), + ('{', {subtypes.NONE}), + ('value', {subtypes.DICTIONARY_KEY, subtypes.DICTIONARY_KEY_PART}), + (':', {subtypes.NONE}), + ('value', {subtypes.DICTIONARY_VALUE}), + ('.', {subtypes.NONE}), + ('lower', {subtypes.NONE}), + ('(', {subtypes.NONE}), + (')', {subtypes.NONE}), + ('}', {subtypes.NONE}), + ], + ]) + + code = textwrap.dedent("""\ + def foo(strs): + return {s: s.lower() for s in strs} + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckFormatTokenSubtypes(llines, [ + [ + ('def', {subtypes.NONE}), + ('foo', {subtypes.FUNC_DEF}), + ('(', {subtypes.NONE}), + ('strs', { + subtypes.NONE, + subtypes.PARAMETER_START, + subtypes.PARAMETER_STOP, + }), + (')', {subtypes.NONE}), + (':', {subtypes.NONE}), + ], + [ + ('return', {subtypes.NONE}), + ('{', {subtypes.NONE}), + ('s', { + subtypes.DICTIONARY_KEY, subtypes.DICTIONARY_KEY_PART, + subtypes.COMP_EXPR + }), + (':', {subtypes.COMP_EXPR}), + ('s', {subtypes.DICTIONARY_VALUE, subtypes.COMP_EXPR}), + ('.', {subtypes.COMP_EXPR}), + ('lower', {subtypes.COMP_EXPR}), + ('(', {subtypes.COMP_EXPR}), + (')', {subtypes.COMP_EXPR}), + ('for', { + subtypes.DICT_SET_GENERATOR, + subtypes.COMP_FOR, + }), + ('s', {subtypes.COMP_FOR}), + ('in', {subtypes.COMP_FOR}), + ('strs', {subtypes.COMP_FOR}), + ('}', {subtypes.NONE}), + ], + ]) + + code = textwrap.dedent("""\ + def foo(strs): + return {c: c.lower() for s in strs for c in s} + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckFormatTokenSubtypes(llines, [ + [ + ('def', {subtypes.NONE}), + ('foo', {subtypes.FUNC_DEF}), + ('(', {subtypes.NONE}), + ('strs', { + subtypes.NONE, + subtypes.PARAMETER_START, + subtypes.PARAMETER_STOP, + }), + (')', {subtypes.NONE}), + (':', {subtypes.NONE}), + ], + [ + ('return', {subtypes.NONE}), + ('{', {subtypes.NONE}), + ('c', { + subtypes.DICTIONARY_KEY, subtypes.DICTIONARY_KEY_PART, + subtypes.COMP_EXPR + }), + (':', {subtypes.COMP_EXPR}), + ('c', {subtypes.DICTIONARY_VALUE, subtypes.COMP_EXPR}), + ('.', {subtypes.COMP_EXPR}), + ('lower', {subtypes.COMP_EXPR}), + ('(', {subtypes.COMP_EXPR}), + (')', {subtypes.COMP_EXPR}), + ('for', { + subtypes.DICT_SET_GENERATOR, + subtypes.COMP_FOR, + subtypes.COMP_EXPR, + }), + ('s', {subtypes.COMP_FOR, subtypes.COMP_EXPR}), + ('in', {subtypes.COMP_FOR, subtypes.COMP_EXPR}), + ('strs', {subtypes.COMP_FOR, subtypes.COMP_EXPR}), + ('for', { + subtypes.DICT_SET_GENERATOR, + subtypes.COMP_FOR, + }), + ('c', {subtypes.COMP_FOR}), + ('in', {subtypes.COMP_FOR}), + ('s', {subtypes.COMP_FOR}), + ('}', {subtypes.NONE}), + ], + ]) def testUnaryNotOperator(self): code = textwrap.dedent("""\ not a - """) - uwlines = self._ParseAndUnwrap(code) - self._CheckFormatTokenSubtypes(uwlines, [ - [('not', {format_token.Subtype.UNARY_OPERATOR}), - ('a', [format_token.Subtype.NONE])] - ]) # yapf: disable + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckFormatTokenSubtypes(llines, [[('not', {subtypes.UNARY_OPERATOR}), + ('a', {subtypes.NONE})]]) def testBitwiseOperators(self): code = textwrap.dedent("""\ x = ((a | (b ^ 3) & c) << 3) >> 1 """) - uwlines = self._ParseAndUnwrap(code) - self._CheckFormatTokenSubtypes(uwlines, [ - [('x', [format_token.Subtype.NONE]), - ('=', {format_token.Subtype.ASSIGN_OPERATOR}), - ('(', [format_token.Subtype.NONE]), - ('(', [format_token.Subtype.NONE]), - ('a', [format_token.Subtype.NONE]), - ('|', {format_token.Subtype.BINARY_OPERATOR}), - ('(', [format_token.Subtype.NONE]), - ('b', [format_token.Subtype.NONE]), - ('^', {format_token.Subtype.BINARY_OPERATOR}), - ('3', [format_token.Subtype.NONE]), - (')', [format_token.Subtype.NONE]), - ('&', {format_token.Subtype.BINARY_OPERATOR}), - ('c', [format_token.Subtype.NONE]), - (')', [format_token.Subtype.NONE]), - ('<<', {format_token.Subtype.BINARY_OPERATOR}), - ('3', [format_token.Subtype.NONE]), - (')', [format_token.Subtype.NONE]), - ('>>', {format_token.Subtype.BINARY_OPERATOR}), - ('1', [format_token.Subtype.NONE])] - ]) # yapf: disable + llines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckFormatTokenSubtypes(llines, [ + [ + ('x', {subtypes.NONE}), + ('=', {subtypes.ASSIGN_OPERATOR}), + ('(', {subtypes.NONE}), + ('(', {subtypes.NONE}), + ('a', {subtypes.NONE}), + ('|', {subtypes.BINARY_OPERATOR}), + ('(', {subtypes.NONE}), + ('b', {subtypes.NONE}), + ('^', {subtypes.BINARY_OPERATOR}), + ('3', {subtypes.NONE}), + (')', {subtypes.NONE}), + ('&', {subtypes.BINARY_OPERATOR}), + ('c', {subtypes.NONE}), + (')', {subtypes.NONE}), + ('<<', {subtypes.BINARY_OPERATOR}), + ('3', {subtypes.NONE}), + (')', {subtypes.NONE}), + ('>>', {subtypes.BINARY_OPERATOR}), + ('1', {subtypes.NONE}), + ], + ]) + + def testArithmeticOperators(self): + code = textwrap.dedent("""\ + x = ((a + (b - 3) * (1 % c) @ d) / 3) // 1 + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckFormatTokenSubtypes(llines, [ + [ + ('x', {subtypes.NONE}), + ('=', {subtypes.ASSIGN_OPERATOR}), + ('(', {subtypes.NONE}), + ('(', {subtypes.NONE}), + ('a', {subtypes.NONE}), + ('+', {subtypes.BINARY_OPERATOR}), + ('(', {subtypes.NONE}), + ('b', {subtypes.NONE}), + ('-', { + subtypes.BINARY_OPERATOR, + subtypes.SIMPLE_EXPRESSION, + }), + ('3', {subtypes.NONE}), + (')', {subtypes.NONE}), + ('*', {subtypes.BINARY_OPERATOR}), + ('(', {subtypes.NONE}), + ('1', {subtypes.NONE}), + ('%', { + subtypes.BINARY_OPERATOR, + subtypes.SIMPLE_EXPRESSION, + }), + ('c', {subtypes.NONE}), + (')', {subtypes.NONE}), + ('@', {subtypes.BINARY_OPERATOR}), + ('d', {subtypes.NONE}), + (')', {subtypes.NONE}), + ('/', {subtypes.BINARY_OPERATOR}), + ('3', {subtypes.NONE}), + (')', {subtypes.NONE}), + ('//', {subtypes.BINARY_OPERATOR}), + ('1', {subtypes.NONE}), + ], + ]) def testSubscriptColon(self): code = textwrap.dedent("""\ x[0:42:1] - """) - uwlines = self._ParseAndUnwrap(code) - self._CheckFormatTokenSubtypes(uwlines, [ - [('x', [format_token.Subtype.NONE]), - ('[', {format_token.Subtype.SUBSCRIPT_BRACKET}), - ('0', [format_token.Subtype.NONE]), - (':', {format_token.Subtype.SUBSCRIPT_COLON}), - ('42', [format_token.Subtype.NONE]), - (':', {format_token.Subtype.SUBSCRIPT_COLON}), - ('1', [format_token.Subtype.NONE]), - (']', {format_token.Subtype.SUBSCRIPT_BRACKET})] - ]) # yapf: disable + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckFormatTokenSubtypes(llines, [ + [ + ('x', {subtypes.NONE}), + ('[', {subtypes.SUBSCRIPT_BRACKET}), + ('0', {subtypes.NONE}), + (':', {subtypes.SUBSCRIPT_COLON}), + ('42', {subtypes.NONE}), + (':', {subtypes.SUBSCRIPT_COLON}), + ('1', {subtypes.NONE}), + (']', {subtypes.SUBSCRIPT_BRACKET}), + ], + ]) def testFunctionCallWithStarExpression(self): code = textwrap.dedent("""\ [a, *b] - """) - uwlines = self._ParseAndUnwrap(code) - self._CheckFormatTokenSubtypes(uwlines, [ - [('[', [format_token.Subtype.NONE]), - ('a', [format_token.Subtype.NONE]), - (',', [format_token.Subtype.NONE]), - ('*', {format_token.Subtype.UNARY_OPERATOR}), - ('b', [format_token.Subtype.NONE]), - (']', [format_token.Subtype.NONE])] - ]) # yapf: disable + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckFormatTokenSubtypes(llines, [ + [ + ('[', {subtypes.NONE}), + ('a', {subtypes.NONE}), + (',', {subtypes.NONE}), + ('*', { + subtypes.UNARY_OPERATOR, + subtypes.VARARGS_STAR, + }), + ('b', {subtypes.NONE}), + (']', {subtypes.NONE}), + ], + ]) if __name__ == '__main__': diff --git a/yapftests/unwrapped_line_test.py b/yapftests/unwrapped_line_test.py deleted file mode 100644 index d3e988944..000000000 --- a/yapftests/unwrapped_line_test.py +++ /dev/null @@ -1,110 +0,0 @@ -# Copyright 2015-2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Tests for yapf.unwrapped_line.""" - -import textwrap -import unittest - -from lib2to3 import pytree -from lib2to3.pgen2 import token - -from yapf.yapflib import comment_splicer -from yapf.yapflib import format_token -from yapf.yapflib import pytree_unwrapper -from yapf.yapflib import pytree_utils -from yapf.yapflib import split_penalty -from yapf.yapflib import subtype_assigner -from yapf.yapflib import unwrapped_line - - -def _MakeFormatTokenLeaf(token_type, token_value): - return format_token.FormatToken(pytree.Leaf(token_type, token_value)) - - -def _MakeFormatTokenList(token_type_values): - return [_MakeFormatTokenLeaf(token_type, token_value) - for token_type, token_value in token_type_values] - - -class UnwrappedLineBasicTest(unittest.TestCase): - - def testConstruction(self): - toks = _MakeFormatTokenList([(token.DOT, '.'), (token.VBAR, '|')]) - uwl = unwrapped_line.UnwrappedLine(20, toks) - self.assertEqual(20, uwl.depth) - self.assertEqual(['DOT', 'VBAR'], [tok.name for tok in uwl.tokens]) - - def testFirstLast(self): - toks = _MakeFormatTokenList([(token.DOT, '.'), (token.LPAR, '('), - (token.VBAR, '|')]) - uwl = unwrapped_line.UnwrappedLine(20, toks) - self.assertEqual(20, uwl.depth) - self.assertEqual('DOT', uwl.first.name) - self.assertEqual('VBAR', uwl.last.name) - - def testAsCode(self): - toks = _MakeFormatTokenList([(token.DOT, '.'), (token.LPAR, '('), - (token.VBAR, '|')]) - uwl = unwrapped_line.UnwrappedLine(2, toks) - self.assertEqual(' . ( |', uwl.AsCode()) - - def testAppendToken(self): - uwl = unwrapped_line.UnwrappedLine(0) - uwl.AppendToken(_MakeFormatTokenLeaf(token.LPAR, '(')) - uwl.AppendToken(_MakeFormatTokenLeaf(token.RPAR, ')')) - self.assertEqual(['LPAR', 'RPAR'], [tok.name for tok in uwl.tokens]) - - def testAppendNode(self): - uwl = unwrapped_line.UnwrappedLine(0) - uwl.AppendNode(pytree.Leaf(token.LPAR, '(')) - uwl.AppendNode(pytree.Leaf(token.RPAR, ')')) - self.assertEqual(['LPAR', 'RPAR'], [tok.name for tok in uwl.tokens]) - - -class UnwrappedLineFormattingInformationTest(unittest.TestCase): - - def _ParseAndUnwrap(self, code): - tree = pytree_utils.ParseCodeToTree(code) - comment_splicer.SpliceComments(tree) - subtype_assigner.AssignSubtypes(tree) - split_penalty.ComputeSplitPenalties(tree) - - uwlines = pytree_unwrapper.UnwrapPyTree(tree) - for i, uwline in enumerate(uwlines): - uwlines[i] = unwrapped_line.UnwrappedLine( - uwline.depth, [ft for ft in uwline.tokens - if ft.name not in pytree_utils.NONSEMANTIC_TOKENS]) - return uwlines - - def testFuncDef(self): - code = textwrap.dedent(r''' - def f(a, b): - pass - ''') - uwlines = self._ParseAndUnwrap(code) - uwlines[0].CalculateFormattingInformation() - - f = uwlines[0].tokens[1] - self.assertFalse(f.can_break_before) - self.assertFalse(f.must_break_before) - self.assertEqual(f.split_penalty, split_penalty.UNBREAKABLE) - - lparen = uwlines[0].tokens[2] - self.assertFalse(lparen.can_break_before) - self.assertFalse(lparen.must_break_before) - self.assertEqual(lparen.split_penalty, split_penalty.UNBREAKABLE) - - -if __name__ == '__main__': - unittest.main() diff --git a/yapftests/utils.py b/yapftests/utils.py new file mode 100644 index 000000000..e91752b8d --- /dev/null +++ b/yapftests/utils.py @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Utilities for tests.""" + +import contextlib +import io +import os +import sys +import tempfile + + +@contextlib.contextmanager +def stdout_redirector(stream): # pylint: disable=invalid-name + old_stdout = sys.stdout + sys.stdout = stream + try: + yield + finally: + sys.stdout = old_stdout + + +# NamedTemporaryFile is useless because on Windows the temporary file would be +# created with O_TEMPORARY, which would not allow the file to be opened a +# second time, even by the same process, unless the same flag is used. +# Thus we provide a simplified version ourselves. +# +# Note: returns a tuple of (io.file_obj, file_path), instead of a file_obj with +# a .name attribute +# +# Note: `buffering` is set to -1 despite documentation of NamedTemporaryFile +# says None. This is probably a problem with the python documentation. +@contextlib.contextmanager +def NamedTempFile(mode='w+b', + buffering=-1, + encoding=None, + errors=None, + newline=None, + suffix=None, + prefix=None, + dirname=None, + text=False): + """Context manager creating a new temporary file in text mode.""" + (fd, fname) = tempfile.mkstemp( + suffix=suffix, prefix=prefix, dir=dirname, text=text) + f = io.open( + fd, + mode=mode, + buffering=buffering, + encoding=encoding, + errors=errors, + newline=newline) + yield f, fname + f.close() + os.remove(fname) + + +@contextlib.contextmanager +def TempFileContents(dirname, + contents, + encoding='utf-8', + newline='', + suffix=None): + # Note: NamedTempFile properly handles unicode encoding when using mode='w' + with NamedTempFile( + dirname=dirname, + mode='w', + encoding=encoding, + newline=newline, + suffix=suffix) as (f, fname): + f.write(contents) + f.flush() + yield fname diff --git a/yapftests/yapf_test.py b/yapftests/yapf_test.py index 449a4b5ff..a9ca01188 100644 --- a/yapftests/yapf_test.py +++ b/yapftests/yapf_test.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2015-2016 Google Inc. All Rights Reserved. +# Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -23,110 +23,95 @@ import tempfile import textwrap import unittest +from io import StringIO -from yapf.yapflib import py3compat +from yapf_third_party._ylib2to3.pgen2 import tokenize + +from yapf.yapflib import errors +from yapf.yapflib import style from yapf.yapflib import yapf_api -ROOT_DIR = os.path.dirname(os.path.abspath(os.path.dirname(__file__))) +from yapftests import utils +from yapftests import yapf_test_helper -# Verification is turned off by default, but want to enable it for testing. -YAPF_BINARY = [sys.executable, '-m', 'yapf', '--verify', '--no-local-style'] +YAPF_BINARY = [sys.executable, '-m', 'yapf', '--no-local-style'] -class FormatCodeTest(unittest.TestCase): +class FormatCodeTest(yapf_test_helper.YAPFTest): def _Check(self, unformatted_code, expected_formatted_code): formatted_code, _ = yapf_api.FormatCode( - unformatted_code, style_config='chromium') - self.assertEqual(expected_formatted_code, formatted_code) + unformatted_code, style_config='yapf') + self.assertCodeEqual(expected_formatted_code, formatted_code) def testSimple(self): - unformatted_code = textwrap.dedent(u"""\ + unformatted_code = textwrap.dedent("""\ print('foo') - """) + """) self._Check(unformatted_code, unformatted_code) def testNoEndingNewline(self): - unformatted_code = textwrap.dedent(u"""\ + unformatted_code = textwrap.dedent("""\ if True: pass""") - expected_formatted_code = textwrap.dedent(u"""\ + expected_formatted_code = textwrap.dedent("""\ if True: pass - """) + """) self._Check(unformatted_code, expected_formatted_code) -class FormatFileTest(unittest.TestCase): +class FormatFileTest(yapf_test_helper.YAPFTest): - def setUp(self): + def setUp(self): # pylint: disable=g-missing-super-call self.test_tmpdir = tempfile.mkdtemp() - def tearDown(self): + def tearDown(self): # pylint: disable=g-missing-super-call shutil.rmtree(self.test_tmpdir) - def assertCodeEqual(self, expected_code, code): - if code != expected_code: - msg = 'Code format mismatch:\n' - msg += 'Expected:\n >' - msg += '\n > '.join(expected_code.splitlines()) - msg += '\nActual:\n >' - msg += '\n > '.join(code.splitlines()) - # TODO(sbc): maybe using difflib here to produce easy to read deltas? - self.fail(msg) - - def _MakeTempFileWithContents(self, filename, contents): - path = os.path.join(self.test_tmpdir, filename) - with open(path, 'w') as f: - f.write(contents) - return path - def testFormatFile(self): - unformatted_code = textwrap.dedent(u"""\ + unformatted_code = textwrap.dedent("""\ if True: pass - """) - file1 = self._MakeTempFileWithContents('testfile1.py', unformatted_code) - - expected_formatted_code_pep8 = textwrap.dedent(u"""\ + """) + expected_formatted_code_pep8 = textwrap.dedent("""\ if True: pass - """) - expected_formatted_code_chromium = textwrap.dedent(u"""\ + """) + expected_formatted_code_yapf = textwrap.dedent("""\ if True: pass - """) - - formatted_code = yapf_api.FormatFile(file1, style_config='pep8')[0] - self.assertCodeEqual(expected_formatted_code_pep8, formatted_code) + """) + with utils.TempFileContents(self.test_tmpdir, unformatted_code) as filepath: + formatted_code, _, _ = yapf_api.FormatFile(filepath, style_config='pep8') + self.assertCodeEqual(expected_formatted_code_pep8, formatted_code) - formatted_code = yapf_api.FormatFile(file1, style_config='chromium')[0] - self.assertCodeEqual(expected_formatted_code_chromium, formatted_code) + formatted_code, _, _ = yapf_api.FormatFile(filepath, style_config='yapf') + self.assertCodeEqual(expected_formatted_code_yapf, formatted_code) def testDisableLinesPattern(self): - unformatted_code = textwrap.dedent(u"""\ + unformatted_code = textwrap.dedent("""\ if a: b # yapf: disable if f: g if h: i - """) - file1 = self._MakeTempFileWithContents('testfile1.py', unformatted_code) - - expected_formatted_code = textwrap.dedent(u"""\ + """) + expected_formatted_code = textwrap.dedent("""\ if a: b # yapf: disable if f: g if h: i - """) - formatted_code = yapf_api.FormatFile(file1, style_config='pep8')[0] - self.assertCodeEqual(expected_formatted_code, formatted_code) + """) + with utils.TempFileContents(self.test_tmpdir, unformatted_code) as filepath: + formatted_code, _, _ = yapf_api.FormatFile(filepath, style_config='pep8') + self.assertCodeEqual(expected_formatted_code, formatted_code) def testDisableAndReenableLinesPattern(self): - unformatted_code = textwrap.dedent(u"""\ + unformatted_code = textwrap.dedent("""\ if a: b # yapf: disable @@ -134,10 +119,8 @@ def testDisableAndReenableLinesPattern(self): # yapf: enable if h: i - """) - file1 = self._MakeTempFileWithContents('testfile1.py', unformatted_code) - - expected_formatted_code = textwrap.dedent(u"""\ + """) + expected_formatted_code = textwrap.dedent("""\ if a: b # yapf: disable @@ -145,12 +128,36 @@ def testDisableAndReenableLinesPattern(self): # yapf: enable if h: i - """) - formatted_code = yapf_api.FormatFile(file1, style_config='pep8')[0] - self.assertCodeEqual(expected_formatted_code, formatted_code) + """) + with utils.TempFileContents(self.test_tmpdir, unformatted_code) as filepath: + formatted_code, _, _ = yapf_api.FormatFile(filepath, style_config='pep8') + self.assertCodeEqual(expected_formatted_code, formatted_code) + + def testFmtOnOff(self): + unformatted_code = textwrap.dedent("""\ + if a: b + + # fmt: off + if f: g + # fmt: on + + if h: i + """) + expected_formatted_code = textwrap.dedent("""\ + if a: b + + # fmt: off + if f: g + # fmt: on + + if h: i + """) + with utils.TempFileContents(self.test_tmpdir, unformatted_code) as filepath: + formatted_code, _, _ = yapf_api.FormatFile(filepath, style_config='pep8') + self.assertCodeEqual(expected_formatted_code, formatted_code) def testDisablePartOfMultilineComment(self): - unformatted_code = textwrap.dedent(u"""\ + unformatted_code = textwrap.dedent("""\ if a: b # This is a multiline comment that disables YAPF. @@ -160,10 +167,8 @@ def testDisablePartOfMultilineComment(self): # This is a multiline comment that enables YAPF. if h: i - """) - file1 = self._MakeTempFileWithContents('testfile1.py', unformatted_code) - - expected_formatted_code = textwrap.dedent(u"""\ + """) + expected_formatted_code = textwrap.dedent("""\ if a: b # This is a multiline comment that disables YAPF. @@ -173,9 +178,10 @@ def testDisablePartOfMultilineComment(self): # This is a multiline comment that enables YAPF. if h: i - """) - formatted_code = yapf_api.FormatFile(file1, style_config='pep8')[0] - self.assertCodeEqual(expected_formatted_code, formatted_code) + """) + with utils.TempFileContents(self.test_tmpdir, unformatted_code) as filepath: + formatted_code, _, _ = yapf_api.FormatFile(filepath, style_config='pep8') + self.assertCodeEqual(expected_formatted_code, formatted_code) code = textwrap.dedent("""\ def foo_function(): @@ -188,73 +194,86 @@ def foo_function(): ) # yapf: enable - """) - file1 = self._MakeTempFileWithContents('testfile1.py', code) - formatted_code = yapf_api.FormatFile(file1, style_config='pep8')[0] - self.assertCodeEqual(code, formatted_code) + """) + with utils.TempFileContents(self.test_tmpdir, code) as filepath: + formatted_code, _, _ = yapf_api.FormatFile(filepath, style_config='pep8') + self.assertCodeEqual(code, formatted_code) + + def testEnabledDisabledSameComment(self): + code = textwrap.dedent("""\ + # yapf: disable + a(bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb, ccccccccccccccccccccccccccccccc, ddddddddddddddddddddddd, eeeeeeeeeeeeeeeeeeeeeeeeeee) + # yapf: enable + # yapf: disable + a(bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb, ccccccccccccccccccccccccccccccc, ddddddddddddddddddddddd, eeeeeeeeeeeeeeeeeeeeeeeeeee) + # yapf: enable + """) # noqa + with utils.TempFileContents(self.test_tmpdir, code) as filepath: + formatted_code, _, _ = yapf_api.FormatFile(filepath, style_config='pep8') + self.assertCodeEqual(code, formatted_code) def testFormatFileLinesSelection(self): - unformatted_code = textwrap.dedent(u"""\ + unformatted_code = textwrap.dedent("""\ if a: b if f: g if h: i - """) - file1 = self._MakeTempFileWithContents('testfile1.py', unformatted_code) - - expected_formatted_code_lines1and2 = textwrap.dedent(u"""\ + """) + expected_formatted_code_lines1and2 = textwrap.dedent("""\ if a: b if f: g if h: i - """) - formatted_code = yapf_api.FormatFile( - file1, style_config='pep8', lines=[(1, 2)])[0] - self.assertCodeEqual(expected_formatted_code_lines1and2, formatted_code) - - expected_formatted_code_lines3 = textwrap.dedent(u"""\ + """) + expected_formatted_code_lines3 = textwrap.dedent("""\ if a: b if f: g if h: i - """) - formatted_code = yapf_api.FormatFile( - file1, style_config='pep8', lines=[(3, 3)])[0] - self.assertCodeEqual(expected_formatted_code_lines3, formatted_code) + """) + with utils.TempFileContents(self.test_tmpdir, unformatted_code) as filepath: + formatted_code, _, _ = yapf_api.FormatFile( + filepath, style_config='pep8', lines=[(1, 2)]) + self.assertCodeEqual(expected_formatted_code_lines1and2, formatted_code) + formatted_code, _, _ = yapf_api.FormatFile( + filepath, style_config='pep8', lines=[(3, 3)]) + self.assertCodeEqual(expected_formatted_code_lines3, formatted_code) def testFormatFileDiff(self): - unformatted_code = textwrap.dedent(u"""\ + unformatted_code = textwrap.dedent("""\ if True: pass - """) - file1 = self._MakeTempFileWithContents('testfile1.py', unformatted_code) - diff = yapf_api.FormatFile(file1, print_diff=True)[0] - self.assertTrue(u'+ pass' in diff) + """) + with utils.TempFileContents(self.test_tmpdir, unformatted_code) as filepath: + diff, _, _ = yapf_api.FormatFile(filepath, print_diff=True) + self.assertIn('+ pass', diff) def testFormatFileInPlace(self): unformatted_code = 'True==False\n' formatted_code = 'True == False\n' - file1 = self._MakeTempFileWithContents('testfile1.py', unformatted_code) - result, _, _ = yapf_api.FormatFile(file1, in_place=True) - self.assertEqual(result, None) - with open(file1) as f: - self.assertCodeEqual(formatted_code, f.read()) - - self.assertRaises( - ValueError, yapf_api.FormatFile, file1, in_place=True, print_diff=True) + with utils.TempFileContents(self.test_tmpdir, unformatted_code) as filepath: + result, _, _ = yapf_api.FormatFile(filepath, in_place=True) + self.assertEqual(result, None) + with open(filepath) as fd: + self.assertCodeEqual(formatted_code, fd.read()) + + self.assertRaises( + ValueError, + yapf_api.FormatFile, + filepath, + in_place=True, + print_diff=True) def testNoFile(self): - stream = py3compat.StringIO() - handler = logging.StreamHandler(stream) - logger = logging.getLogger('mylogger') - logger.addHandler(handler) - self.assertRaises( - IOError, yapf_api.FormatFile, 'not_a_file.py', logger=logger.error) - self.assertEqual(stream.getvalue(), - "[Errno 2] No such file or directory: 'not_a_file.py'\n") + with self.assertRaises(IOError) as context: + yapf_api.FormatFile('not_a_file.py') + + self.assertEqual( + str(context.exception), + "[Errno 2] No such file or directory: 'not_a_file.py'") def testCommentsUnformatted(self): code = textwrap.dedent("""\ @@ -263,10 +282,10 @@ def testCommentsUnformatted(self): 'one', # quark 'two'] # yapf: disable - """) - file1 = self._MakeTempFileWithContents('testfile1.py', code) - formatted_code = yapf_api.FormatFile(file1, style_config='pep8')[0] - self.assertCodeEqual(code, formatted_code) + """) + with utils.TempFileContents(self.test_tmpdir, code) as filepath: + formatted_code, _, _ = yapf_api.FormatFile(filepath, style_config='pep8') + self.assertCodeEqual(code, formatted_code) def testDisabledHorizontalFormattingOnNewLine(self): code = textwrap.dedent("""\ @@ -274,19 +293,56 @@ def testDisabledHorizontalFormattingOnNewLine(self): a = [ 1] # yapf: enable - """) - file1 = self._MakeTempFileWithContents('testfile1.py', code) - formatted_code = yapf_api.FormatFile(file1, style_config='pep8')[0] - self.assertCodeEqual(code, formatted_code) + """) + with utils.TempFileContents(self.test_tmpdir, code) as filepath: + formatted_code, _, _ = yapf_api.FormatFile(filepath, style_config='pep8') + self.assertCodeEqual(code, formatted_code) + + def testSplittingSemicolonStatements(self): + unformatted_code = textwrap.dedent("""\ + def f(): + x = y + 42 ; z = n * 42 + if True: a += 1 ; b += 1; c += 1 + """) + expected_formatted_code = textwrap.dedent("""\ + def f(): + x = y + 42 + z = n * 42 + if True: + a += 1 + b += 1 + c += 1 + """) + with utils.TempFileContents(self.test_tmpdir, unformatted_code) as filepath: + formatted_code, _, _ = yapf_api.FormatFile(filepath, style_config='pep8') + self.assertCodeEqual(expected_formatted_code, formatted_code) + + def testSemicolonStatementsDisabled(self): + unformatted_code = textwrap.dedent("""\ + def f(): + x = y + 42 ; z = n * 42 # yapf: disable + if True: a += 1 ; b += 1; c += 1 + """) + expected_formatted_code = textwrap.dedent("""\ + def f(): + x = y + 42 ; z = n * 42 # yapf: disable + if True: + a += 1 + b += 1 + c += 1 + """) + with utils.TempFileContents(self.test_tmpdir, unformatted_code) as filepath: + formatted_code, _, _ = yapf_api.FormatFile(filepath, style_config='pep8') + self.assertCodeEqual(expected_formatted_code, formatted_code) def testDisabledSemiColonSeparatedStatements(self): code = textwrap.dedent("""\ # yapf: disable if True: a ; b - """) - file1 = self._MakeTempFileWithContents('testfile1.py', code) - formatted_code = yapf_api.FormatFile(file1, style_config='pep8')[0] - self.assertCodeEqual(code, formatted_code) + """) + with utils.TempFileContents(self.test_tmpdir, code) as filepath: + formatted_code, _, _ = yapf_api.FormatFile(filepath, style_config='pep8') + self.assertCodeEqual(code, formatted_code) def testDisabledMultilineStringInDictionary(self): code = textwrap.dedent("""\ @@ -302,24 +358,53 @@ def testDisabledMultilineStringInDictionary(self): ''', }, ] - """) - file1 = self._MakeTempFileWithContents('testfile1.py', code) - formatted_code = yapf_api.FormatFile(file1, style_config='chromium')[0] - self.assertCodeEqual(code, formatted_code) + """) + with utils.TempFileContents(self.test_tmpdir, code) as filepath: + formatted_code, _, _ = yapf_api.FormatFile(filepath, style_config='yapf') + self.assertCodeEqual(code, formatted_code) + + def testDisabledWithPrecedingText(self): + code = textwrap.dedent("""\ + # TODO(fix formatting): yapf: disable + + A = [ + { + "aaaaaaaaaaaaaaaaaaa": ''' + bbbbbbbbbbb: "ccccccccccc" + dddddddddddddd: 1 + eeeeeeee: 0 + ffffffffff: "ggggggg" + ''', + }, + ] + """) + with utils.TempFileContents(self.test_tmpdir, code) as filepath: + formatted_code, _, _ = yapf_api.FormatFile(filepath, style_config='yapf') + self.assertCodeEqual(code, formatted_code) + def testCRLFLineEnding(self): + code = 'class _():\r\n pass\r\n' + with utils.TempFileContents(self.test_tmpdir, code) as filepath: + formatted_code, _, _ = yapf_api.FormatFile(filepath, style_config='yapf') + self.assertCodeEqual(code, formatted_code) -class CommandLineTest(unittest.TestCase): + +class CommandLineTest(yapf_test_helper.YAPFTest): """Test how calling yapf from the command line acts.""" @classmethod - def setUpClass(cls): + def setUpClass(cls): # pylint: disable=g-missing-super-call cls.test_tmpdir = tempfile.mkdtemp() @classmethod - def tearDownClass(cls): + def tearDownClass(cls): # pylint: disable=g-missing-super-call shutil.rmtree(cls.test_tmpdir) - def assertYapfReformats(self, unformatted, expected, extra_options=None): + def assertYapfReformats(self, + unformatted, + expected, + extra_options=None, + env=None): """Check that yapf reformats the given code as expected. Invokes yapf in a subprocess, piping the unformatted code into its stdin. @@ -329,172 +414,196 @@ def assertYapfReformats(self, unformatted, expected, extra_options=None): unformatted: unformatted code - input to yapf expected: expected formatted code at the output of yapf extra_options: iterable of extra command-line options to pass to yapf + env: dict of environment variables. """ cmdline = YAPF_BINARY + (extra_options or []) p = subprocess.Popen( cmdline, stdout=subprocess.PIPE, stdin=subprocess.PIPE, - stderr=subprocess.PIPE) - reformatted_code, stderrdata = p.communicate(unformatted.encode('utf-8')) + stderr=subprocess.PIPE, + env=env) + reformatted_code, stderrdata = p.communicate( + unformatted.encode('utf-8-sig')) self.assertEqual(stderrdata, b'') - self.assertEqual(reformatted_code.decode('utf-8'), expected) - - def testUnicodeEncodingPipedToFile(self): - unformatted_code = textwrap.dedent(u"""\ - def foo(): - print('⇒') - """) - - with tempfile.NamedTemporaryFile( - suffix='.py', dir=self.test_tmpdir) as outfile: - with tempfile.NamedTemporaryFile( - suffix='.py', dir=self.test_tmpdir) as testfile: - testfile.write(unformatted_code.encode('UTF-8')) - subprocess.check_call( - YAPF_BINARY + ['--diff', testfile.name], stdout=outfile) + self.assertMultiLineEqual(reformatted_code.decode('utf-8'), expected) def testInPlaceReformatting(self): - unformatted_code = textwrap.dedent(u"""\ + unformatted_code = textwrap.dedent("""\ def foo(): x = 37 - """) - expected_formatted_code = textwrap.dedent(u"""\ + """) + expected_formatted_code = textwrap.dedent("""\ def foo(): x = 37 - """) - - with tempfile.NamedTemporaryFile( - suffix='.py', dir=self.test_tmpdir) as testfile: - testfile.write(unformatted_code.encode('UTF-8')) - testfile.seek(0) - - p = subprocess.Popen(YAPF_BINARY + ['--in-place', testfile.name]) + """) + with utils.TempFileContents( + self.test_tmpdir, unformatted_code, suffix='.py') as filepath: + p = subprocess.Popen(YAPF_BINARY + ['--in-place', filepath]) p.wait() - - with io.open(testfile.name, mode='r', newline='') as fd: + with io.open(filepath, mode='r', newline='') as fd: reformatted_code = fd.read() - self.assertEqual(reformatted_code, expected_formatted_code) def testInPlaceReformattingBlank(self): - unformatted_code = u'\n\n' - expected_formatted_code = u'\n' - - with tempfile.NamedTemporaryFile( - suffix='.py', dir=self.test_tmpdir) as testfile: - testfile.write(unformatted_code.encode('UTF-8')) - testfile.seek(0) - - p = subprocess.Popen(YAPF_BINARY + ['--in-place', testfile.name]) + unformatted_code = '\n\n' + expected_formatted_code = '\n' + with utils.TempFileContents( + self.test_tmpdir, unformatted_code, suffix='.py') as filepath: + p = subprocess.Popen(YAPF_BINARY + ['--in-place', filepath]) p.wait() + with io.open(filepath, mode='r', encoding='utf-8', newline='') as fd: + reformatted_code = fd.read() + self.assertEqual(reformatted_code, expected_formatted_code) - with io.open(testfile.name, mode='r', newline='') as fd: + def testInPlaceReformattingWindowsNewLine(self): + unformatted_code = '\r\n\r\n' + expected_formatted_code = '\r\n' + with utils.TempFileContents( + self.test_tmpdir, unformatted_code, suffix='.py') as filepath: + p = subprocess.Popen(YAPF_BINARY + ['--in-place', filepath]) + p.wait() + with io.open(filepath, mode='r', encoding='utf-8', newline='') as fd: reformatted_code = fd.read() + self.assertEqual(reformatted_code, expected_formatted_code) + def testInPlaceReformattingNoNewLine(self): + unformatted_code = textwrap.dedent('def foo(): x = 37') + expected_formatted_code = textwrap.dedent("""\ + def foo(): + x = 37 + """) + with utils.TempFileContents( + self.test_tmpdir, unformatted_code, suffix='.py') as filepath: + p = subprocess.Popen(YAPF_BINARY + ['--in-place', filepath]) + p.wait() + with io.open(filepath, mode='r', newline='') as fd: + reformatted_code = fd.read() self.assertEqual(reformatted_code, expected_formatted_code) def testInPlaceReformattingEmpty(self): - unformatted_code = u'' - expected_formatted_code = u'' - - with tempfile.NamedTemporaryFile( - suffix='.py', dir=self.test_tmpdir) as testfile: - testfile.write(unformatted_code.encode('UTF-8')) - testfile.seek(0) - - p = subprocess.Popen(YAPF_BINARY + ['--in-place', testfile.name]) + unformatted_code = '' + expected_formatted_code = '' + with utils.TempFileContents( + self.test_tmpdir, unformatted_code, suffix='.py') as filepath: + p = subprocess.Popen(YAPF_BINARY + ['--in-place', filepath]) p.wait() - - with io.open(testfile.name, mode='r', newline='') as fd: + with io.open(filepath, mode='r', encoding='utf-8', newline='') as fd: reformatted_code = fd.read() - self.assertEqual(reformatted_code, expected_formatted_code) + def testPrintModified(self): + for unformatted_code, has_change in [('1==2', True), ('1 == 2', False)]: + with utils.TempFileContents( + self.test_tmpdir, unformatted_code, suffix='.py') as filepath: + output = subprocess.check_output( + YAPF_BINARY + ['--in-place', '--print-modified', filepath], + text=True) + check = self.assertIn if has_change else self.assertNotIn + check(f'Formatted {filepath}', output) + def testReadFromStdin(self): - unformatted_code = textwrap.dedent(u"""\ + unformatted_code = textwrap.dedent("""\ def foo(): x = 37 - """) - expected_formatted_code = textwrap.dedent(u"""\ + """) + expected_formatted_code = textwrap.dedent("""\ def foo(): x = 37 - """) + """) self.assertYapfReformats(unformatted_code, expected_formatted_code) def testReadFromStdinWithEscapedStrings(self): - unformatted_code = textwrap.dedent(u"""\ + unformatted_code = textwrap.dedent("""\ s = "foo\\nbar" - """) - expected_formatted_code = textwrap.dedent(u"""\ + """) + expected_formatted_code = textwrap.dedent("""\ s = "foo\\nbar" - """) + """) self.assertYapfReformats(unformatted_code, expected_formatted_code) - def testSetChromiumStyle(self): - unformatted_code = textwrap.dedent(u"""\ + def testSetYapfStyle(self): + unformatted_code = textwrap.dedent("""\ def foo(): # trail x = 37 - """) - expected_formatted_code = textwrap.dedent(u"""\ + """) + expected_formatted_code = textwrap.dedent("""\ def foo(): # trail x = 37 - """) + """) self.assertYapfReformats( unformatted_code, expected_formatted_code, - extra_options=['--style=chromium']) + extra_options=['--style=yapf']) - def testSetCustomStyleBasedOnChromium(self): - unformatted_code = textwrap.dedent(u"""\ + def testSetCustomStyleBasedOnYapf(self): + unformatted_code = textwrap.dedent("""\ def foo(): # trail x = 37 - """) - expected_formatted_code = textwrap.dedent(u"""\ + """) + expected_formatted_code = textwrap.dedent("""\ def foo(): # trail x = 37 - """) + """) + style_file = textwrap.dedent("""\ + [style] + based_on_style = yapf + spaces_before_comment = 4 + """) + with utils.TempFileContents(self.test_tmpdir, style_file) as stylepath: + self.assertYapfReformats( + unformatted_code, + expected_formatted_code, + extra_options=['--style={0}'.format(stylepath)]) - with tempfile.NamedTemporaryFile(dir=self.test_tmpdir, mode='w') as f: - f.write( - textwrap.dedent('''\ - [style] - based_on_style = chromium - SPACES_BEFORE_COMMENT = 4 - ''')) - f.flush() + def testSetCustomStyleSpacesBeforeComment(self): + unformatted_code = textwrap.dedent("""\ + a_very_long_statement_that_extends_way_beyond # Comment + short # This is a shorter statement + """) + expected_formatted_code = textwrap.dedent("""\ + a_very_long_statement_that_extends_way_beyond # Comment + short # This is a shorter statement + """) # noqa + style_file = textwrap.dedent("""\ + [style] + spaces_before_comment = 15, 20 + """) + with utils.TempFileContents(self.test_tmpdir, style_file) as stylepath: self.assertYapfReformats( unformatted_code, expected_formatted_code, - extra_options=['--style={0}'.format(f.name)]) + extra_options=['--style={0}'.format(stylepath)]) def testReadSingleLineCodeFromStdin(self): - unformatted_code = textwrap.dedent(u"""\ + unformatted_code = textwrap.dedent("""\ if True: pass - """) - expected_formatted_code = textwrap.dedent(u"""\ + """) + expected_formatted_code = textwrap.dedent("""\ if True: pass - """) + """) self.assertYapfReformats(unformatted_code, expected_formatted_code) def testEncodingVerification(self): - unformatted_code = textwrap.dedent(u"""\ + unformatted_code = textwrap.dedent("""\ '''The module docstring.''' # -*- coding: utf-8 -*- def f(): x = 37 - """) + """) - with tempfile.NamedTemporaryFile( - suffix='.py', dir=self.test_tmpdir) as outfile: - with tempfile.NamedTemporaryFile( - suffix='.py', dir=self.test_tmpdir) as testfile: - testfile.write(unformatted_code.encode('utf-8')) - subprocess.check_call( - YAPF_BINARY + ['--diff', testfile.name], stdout=outfile) + with utils.NamedTempFile( + suffix='.py', dirname=self.test_tmpdir) as (out, _): + with utils.TempFileContents( + self.test_tmpdir, unformatted_code, suffix='.py') as filepath: + try: + subprocess.check_call(YAPF_BINARY + ['--diff', filepath], stdout=out) + except subprocess.CalledProcessError as e: + # Indicates the text changed. + self.assertEqual(e.returncode, 1) # pylint: disable=g-assert-in-except # noqa def testReformattingSpecificLines(self): - unformatted_code = textwrap.dedent(u"""\ + unformatted_code = textwrap.dedent("""\ def h(): if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): pass @@ -503,8 +612,8 @@ def h(): def g(): if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): pass - """) - expected_formatted_code = textwrap.dedent(u"""\ + """) # noqa + expected_formatted_code = textwrap.dedent("""\ def h(): if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): @@ -514,7 +623,7 @@ def h(): def g(): if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): pass - """) + """) # noqa # TODO(ambv): the `expected_formatted_code` here is not PEP8 compliant, # raising "E129 visually indented line with same indent as next logical # line" with flake8. @@ -523,8 +632,28 @@ def g(): expected_formatted_code, extra_options=['--lines', '1-2']) + def testOmitFormattingLinesBeforeDisabledFunctionComment(self): + unformatted_code = textwrap.dedent("""\ + import sys + + # Comment + def some_func(x): + x = ["badly" , "formatted","line" ] + """) + expected_formatted_code = textwrap.dedent("""\ + import sys + + # Comment + def some_func(x): + x = ["badly", "formatted", "line"] + """) + self.assertYapfReformats( + unformatted_code, + expected_formatted_code, + extra_options=['--lines', '5-5']) + def testReformattingSkippingLines(self): - unformatted_code = textwrap.dedent(u"""\ + unformatted_code = textwrap.dedent("""\ def h(): if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): pass @@ -534,8 +663,8 @@ def g(): if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): pass # yapf: enable - """) - expected_formatted_code = textwrap.dedent(u"""\ + """) # noqa + expected_formatted_code = textwrap.dedent("""\ def h(): if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): @@ -547,11 +676,11 @@ def g(): if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): pass # yapf: enable - """) + """) # noqa self.assertYapfReformats(unformatted_code, expected_formatted_code) def testReformattingSkippingToEndOfFile(self): - unformatted_code = textwrap.dedent(u"""\ + unformatted_code = textwrap.dedent("""\ def h(): if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): pass @@ -567,8 +696,8 @@ def e(): xxxxxxxxxxxxxxxxxxxxx(yyyyyyyyyyyyy[zzzzz].aaaaaaaa[0]) == 'bbbbbbb'): pass - """) - expected_formatted_code = textwrap.dedent(u"""\ + """) # noqa + expected_formatted_code = textwrap.dedent("""\ def h(): if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): @@ -586,11 +715,11 @@ def e(): xxxxxxxxxxxxxxxxxxxxx(yyyyyyyyyyyyy[zzzzz].aaaaaaaa[0]) == 'bbbbbbb'): pass - """) + """) # noqa self.assertYapfReformats(unformatted_code, expected_formatted_code) def testReformattingSkippingSingleLine(self): - unformatted_code = textwrap.dedent(u"""\ + unformatted_code = textwrap.dedent("""\ def h(): if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): pass @@ -598,8 +727,8 @@ def h(): def g(): if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): # yapf: disable pass - """) - expected_formatted_code = textwrap.dedent(u"""\ + """) # noqa + expected_formatted_code = textwrap.dedent("""\ def h(): if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): @@ -609,41 +738,43 @@ def h(): def g(): if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): # yapf: disable pass - """) + """) # noqa self.assertYapfReformats(unformatted_code, expected_formatted_code) def testDisableWholeDataStructure(self): - unformatted_code = textwrap.dedent(u"""\ + unformatted_code = textwrap.dedent("""\ A = set([ 'hello', 'world', ]) # yapf: disable - """) - expected_formatted_code = textwrap.dedent(u"""\ + """) + expected_formatted_code = textwrap.dedent("""\ A = set([ 'hello', 'world', ]) # yapf: disable - """) + """) self.assertYapfReformats(unformatted_code, expected_formatted_code) def testDisableButAdjustIndentations(self): - unformatted_code = textwrap.dedent(u"""\ + unformatted_code = textwrap.dedent("""\ class SplitPenaltyTest(unittest.TestCase): + def testUnbreakable(self): self._CheckPenalties(tree, [ ]) # yapf: disable - """) - expected_formatted_code = textwrap.dedent(u"""\ + """) + expected_formatted_code = textwrap.dedent("""\ class SplitPenaltyTest(unittest.TestCase): + def testUnbreakable(self): self._CheckPenalties(tree, [ ]) # yapf: disable - """) + """) self.assertYapfReformats(unformatted_code, expected_formatted_code) def testRetainingHorizontalWhitespace(self): - unformatted_code = textwrap.dedent(u"""\ + unformatted_code = textwrap.dedent("""\ def h(): if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): pass @@ -651,8 +782,8 @@ def h(): def g(): if (xxxxxxxxxxxx.yyyyyyyy (zzzzzzzzzzzzz [0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): # yapf: disable pass - """) - expected_formatted_code = textwrap.dedent(u"""\ + """) # noqa + expected_formatted_code = textwrap.dedent("""\ def h(): if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): @@ -662,11 +793,11 @@ def h(): def g(): if (xxxxxxxxxxxx.yyyyyyyy (zzzzzzzzzzzzz [0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): # yapf: disable pass - """) + """) # noqa self.assertYapfReformats(unformatted_code, expected_formatted_code) def testRetainingVerticalWhitespace(self): - unformatted_code = textwrap.dedent(u"""\ + unformatted_code = textwrap.dedent("""\ def h(): if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): pass @@ -677,8 +808,8 @@ def g(): if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): pass - """) - expected_formatted_code = textwrap.dedent(u"""\ + """) # noqa + expected_formatted_code = textwrap.dedent("""\ def h(): if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): @@ -690,13 +821,13 @@ def g(): if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): pass - """) + """) # noqa self.assertYapfReformats( unformatted_code, expected_formatted_code, extra_options=['--lines', '1-2']) - unformatted_code = textwrap.dedent(u"""\ + unformatted_code = textwrap.dedent("""\ if a: b @@ -712,8 +843,8 @@ def g(): #comment # trailing whitespace - """) - expected_formatted_code = textwrap.dedent(u"""\ + """) + expected_formatted_code = textwrap.dedent("""\ if a: b @@ -727,37 +858,59 @@ def g(): #comment # trailing whitespace - """) + """) self.assertYapfReformats( unformatted_code, expected_formatted_code, extra_options=['--lines', '3-3', '--lines', '13-13']) - unformatted_code = textwrap.dedent(u"""\ + unformatted_code = textwrap.dedent("""\ ''' docstring ''' import blah - """) + """) self.assertYapfReformats( unformatted_code, unformatted_code, extra_options=['--lines', '2-2']) + def testVerticalSpacingWithCommentWithContinuationMarkers(self): + unformatted_code = textwrap.dedent("""\ + # \\ + # \\ + # \\ + + x = { + } + """) + expected_formatted_code = textwrap.dedent("""\ + # \\ + # \\ + # \\ + + x = { + } + """) + self.assertYapfReformats( + unformatted_code, + expected_formatted_code, + extra_options=['--lines', '1-1']) + def testRetainingSemicolonsWhenSpecifyingLines(self): unformatted_code = textwrap.dedent("""\ a = line_to_format def f(): x = y + 42; z = n * 42 if True: a += 1 ; b += 1 ; c += 1 - """) + """) expected_formatted_code = textwrap.dedent("""\ a = line_to_format def f(): x = y + 42; z = n * 42 if True: a += 1 ; b += 1 ; c += 1 - """) + """) self.assertYapfReformats( unformatted_code, expected_formatted_code, @@ -773,7 +926,7 @@ def f(): Residence: """+palace["Winter"]+"""
""" - ''') + ''') # noqa expected_formatted_code = textwrap.dedent('''\ foo = 42 def f(): @@ -783,14 +936,14 @@ def f(): Residence: """+palace["Winter"]+"""
""" - ''') + ''') # noqa self.assertYapfReformats( unformatted_code, expected_formatted_code, extra_options=['--lines', '1-1']) def testDisableWhenSpecifyingLines(self): - unformatted_code = textwrap.dedent(u"""\ + unformatted_code = textwrap.dedent("""\ # yapf: disable A = set([ 'hello', @@ -801,8 +954,8 @@ def testDisableWhenSpecifyingLines(self): 'hello', 'world', ]) # yapf: disable - """) - expected_formatted_code = textwrap.dedent(u"""\ + """) + expected_formatted_code = textwrap.dedent("""\ # yapf: disable A = set([ 'hello', @@ -813,14 +966,14 @@ def testDisableWhenSpecifyingLines(self): 'hello', 'world', ]) # yapf: disable - """) + """) self.assertYapfReformats( unformatted_code, expected_formatted_code, extra_options=['--lines', '1-10']) def testDisableFormattingInDataLiteral(self): - unformatted_code = textwrap.dedent(u"""\ + unformatted_code = textwrap.dedent("""\ def horrible(): oh_god() why_would_you() @@ -838,8 +991,8 @@ def still_horrible(): 'that' ] - """) - expected_formatted_code = textwrap.dedent(u"""\ + """) + expected_formatted_code = textwrap.dedent("""\ def horrible(): oh_god() why_would_you() @@ -853,14 +1006,14 @@ def still_horrible(): oh_god() why_would_you() ['do', 'that'] - """) + """) self.assertYapfReformats( unformatted_code, expected_formatted_code, extra_options=['--lines', '14-15']) def testRetainVerticalFormattingBetweenDisabledAndEnabledLines(self): - unformatted_code = textwrap.dedent(u"""\ + unformatted_code = textwrap.dedent("""\ class A(object): def aaaaaaaaaaaaa(self): c = bbbbbbbbb.ccccccccc('challenge', 0, 1, 10) @@ -870,8 +1023,8 @@ def aaaaaaaaaaaaa(self): c.ffffffffffff), gggggggggggg.hhhhhhhhh(c, c.ffffffffffff)) iiiii = jjjjjjjjjjjjjj.iiiii - """) - expected_formatted_code = textwrap.dedent(u"""\ + """) + expected_formatted_code = textwrap.dedent("""\ class A(object): def aaaaaaaaaaaaa(self): c = bbbbbbbbb.ccccccccc('challenge', 0, 1, 10) @@ -879,14 +1032,38 @@ def aaaaaaaaaaaaa(self): 'eeeeeeeeeeeeeeeeeeeeeeeee.%s' % c.ffffffffffff), gggggggggggg.hhhhhhhhh(c, c.ffffffffffff)) iiiii = jjjjjjjjjjjjjj.iiiii - """) + """) # noqa self.assertYapfReformats( unformatted_code, expected_formatted_code, extra_options=['--lines', '4-7']) + def testRetainVerticalFormattingBetweenDisabledLines(self): + unformatted_code = textwrap.dedent("""\ + class A(object): + def aaaaaaaaaaaaa(self): + pass + + + def bbbbbbbbbbbbb(self): # 5 + pass + """) + expected_formatted_code = textwrap.dedent("""\ + class A(object): + def aaaaaaaaaaaaa(self): + pass + + + def bbbbbbbbbbbbb(self): # 5 + pass + """) + self.assertYapfReformats( + unformatted_code, + expected_formatted_code, + extra_options=['--lines', '4-4']) + def testFormatLinesSpecifiedInMiddleOfExpression(self): - unformatted_code = textwrap.dedent(u"""\ + unformatted_code = textwrap.dedent("""\ class A(object): def aaaaaaaaaaaaa(self): c = bbbbbbbbb.ccccccccc('challenge', 0, 1, 10) @@ -896,8 +1073,8 @@ def aaaaaaaaaaaaa(self): c.ffffffffffff), gggggggggggg.hhhhhhhhh(c, c.ffffffffffff)) iiiii = jjjjjjjjjjjjjj.iiiii - """) - expected_formatted_code = textwrap.dedent(u"""\ + """) + expected_formatted_code = textwrap.dedent("""\ class A(object): def aaaaaaaaaaaaa(self): c = bbbbbbbbb.ccccccccc('challenge', 0, 1, 10) @@ -905,29 +1082,29 @@ def aaaaaaaaaaaaa(self): 'eeeeeeeeeeeeeeeeeeeeeeeee.%s' % c.ffffffffffff), gggggggggggg.hhhhhhhhh(c, c.ffffffffffff)) iiiii = jjjjjjjjjjjjjj.iiiii - """) + """) # noqa self.assertYapfReformats( unformatted_code, expected_formatted_code, extra_options=['--lines', '5-6']) def testCommentFollowingMultilineString(self): - unformatted_code = textwrap.dedent(u"""\ + unformatted_code = textwrap.dedent("""\ def foo(): '''First line. Second line. ''' # comment x = '''hello world''' # second comment return 42 # another comment - """) - expected_formatted_code = textwrap.dedent(u"""\ + """) + expected_formatted_code = textwrap.dedent("""\ def foo(): '''First line. Second line. ''' # comment x = '''hello world''' # second comment return 42 # another comment - """) + """) self.assertYapfReformats( unformatted_code, expected_formatted_code, @@ -935,16 +1112,16 @@ def foo(): def testDedentClosingBracket(self): # no line-break on the first argument, not dedenting closing brackets - unformatted_code = textwrap.dedent(u"""\ - def overly_long_function_name(first_argument_on_the_same_line, - second_argument_makes_the_line_too_long): - pass - """) - expected_formatted_code = textwrap.dedent(u"""\ - def overly_long_function_name(first_argument_on_the_same_line, - second_argument_makes_the_line_too_long): + unformatted_code = textwrap.dedent("""\ + def overly_long_function_name(first_argument_on_the_same_line, + second_argument_makes_the_line_too_long): pass """) + expected_formatted_code = textwrap.dedent("""\ + def overly_long_function_name(first_argument_on_the_same_line, + second_argument_makes_the_line_too_long): + pass + """) # noqa self.assertYapfReformats( unformatted_code, expected_formatted_code, @@ -958,24 +1135,24 @@ def overly_long_function_name(first_argument_on_the_same_line, # extra_options=['--style=facebook']) # line-break before the first argument, dedenting closing brackets if set - unformatted_code = textwrap.dedent(u"""\ - def overly_long_function_name( - first_argument_on_the_same_line, - second_argument_makes_the_line_too_long): - pass - """) - expected_formatted_pep8_code = textwrap.dedent(u"""\ - def overly_long_function_name( - first_argument_on_the_same_line, \ -second_argument_makes_the_line_too_long): - pass - """) - expected_formatted_fb_code = textwrap.dedent(u"""\ - def overly_long_function_name( - first_argument_on_the_same_line, second_argument_makes_the_line_too_long - ): + unformatted_code = textwrap.dedent("""\ + def overly_long_function_name( + first_argument_on_the_same_line, + second_argument_makes_the_line_too_long): pass """) + # expected_formatted_pep8_code = textwrap.dedent("""\ + # def overly_long_function_name( + # first_argument_on_the_same_line, + # second_argument_makes_the_line_too_long): + # pass + # """) + expected_formatted_fb_code = textwrap.dedent("""\ + def overly_long_function_name( + first_argument_on_the_same_line, second_argument_makes_the_line_too_long + ): + pass + """) # noqa self.assertYapfReformats( unformatted_code, expected_formatted_fb_code, @@ -988,49 +1165,51 @@ def overly_long_function_name( # extra_options=['--style=pep8']) def testCoalesceBrackets(self): - unformatted_code = textwrap.dedent(u"""\ - some_long_function_name_foo({ - 'first_argument_of_the_thing': id, - 'second_argument_of_the_thing': "some thing"} - )""") - expected_formatted_code = textwrap.dedent(u"""\ + unformatted_code = textwrap.dedent("""\ + some_long_function_name_foo( + { + 'first_argument_of_the_thing': id, + 'second_argument_of_the_thing': "some thing" + } + ) + """) + expected_formatted_code = textwrap.dedent("""\ some_long_function_name_foo({ 'first_argument_of_the_thing': id, 'second_argument_of_the_thing': "some thing" }) - """) - with tempfile.NamedTemporaryFile(dir=self.test_tmpdir, mode='w') as f: + """) + with utils.NamedTempFile(dirname=self.test_tmpdir, mode='w') as (f, name): f.write( - textwrap.dedent('''\ + textwrap.dedent("""\ [style] - based_on_style = facebook column_limit=82 coalesce_brackets = True - ''')) + """)) f.flush() self.assertYapfReformats( unformatted_code, expected_formatted_code, - extra_options=['--style={0}'.format(f.name)]) + extra_options=['--style={0}'.format(name)]) def testPseudoParenSpaces(self): - unformatted_code = textwrap.dedent(u"""\ - def foo(): + unformatted_code = textwrap.dedent("""\ + def foo(): def bar(): return {msg_id: author for author, msg_id in reader} - """) - expected_formatted_code = textwrap.dedent(u"""\ + """) + expected_formatted_code = textwrap.dedent("""\ def foo(): def bar(): return {msg_id: author for author, msg_id in reader} - """) + """) self.assertYapfReformats( unformatted_code, expected_formatted_code, - extra_options=['--lines', '1-1', '--style', 'chromium']) + extra_options=['--lines', '1-1', '--style', 'yapf']) def testMultilineCommentFormattingDisabled(self): - unformatted_code = textwrap.dedent(u"""\ + unformatted_code = textwrap.dedent("""\ # This is a comment FOO = { aaaaaaaa.ZZZ: [ @@ -1043,8 +1222,8 @@ def testMultilineCommentFormattingDisabled(self): ('yyyyy', zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz), '#': lambda x: x # do nothing } - """) - expected_formatted_code = textwrap.dedent(u"""\ + """) + expected_formatted_code = textwrap.dedent("""\ # This is a comment FOO = { aaaaaaaa.ZZZ: [ @@ -1057,64 +1236,820 @@ def testMultilineCommentFormattingDisabled(self): ('yyyyy', zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz), '#': lambda x: x # do nothing } - """) + """) self.assertYapfReformats( unformatted_code, expected_formatted_code, - extra_options=['--lines', '1-1', '--style', 'chromium']) + extra_options=['--lines', '1-1', '--style', 'yapf']) def testTrailingCommentsWithDisabledFormatting(self): - unformatted_code = textwrap.dedent(u"""\ + unformatted_code = textwrap.dedent("""\ import os SCOPES = [ 'hello world' # This is a comment. ] - """) - expected_formatted_code = textwrap.dedent(u"""\ + """) + expected_formatted_code = textwrap.dedent("""\ import os SCOPES = [ 'hello world' # This is a comment. ] - """) + """) self.assertYapfReformats( unformatted_code, expected_formatted_code, - extra_options=['--lines', '1-1', '--style', 'chromium']) + extra_options=['--lines', '1-1', '--style', 'yapf']) def testUseTabs(self): - unformatted_code = textwrap.dedent(u"""\ + unformatted_code = textwrap.dedent("""\ def foo_function(): if True: pass - """) - expected_formatted_code = textwrap.dedent(u"""\ + """) + expected_formatted_code = """\ +def foo_function(): + if True: + pass +""" # noqa: W191,E101 + style_contents = textwrap.dedent("""\ + [style] + based_on_style = yapf + use_tabs = true + indent_width = 1 + """) + with utils.TempFileContents(self.test_tmpdir, style_contents) as stylepath: + self.assertYapfReformats( + unformatted_code, + expected_formatted_code, + extra_options=['--style={0}'.format(stylepath)]) + + def testUseTabsWith(self): + unformatted_code = """\ +def f(): + return ['hello', 'world',] +""" + expected_formatted_code = """\ +def f(): + return [ + 'hello', + 'world', + ] +""" # noqa: W191,E101 + style_contents = textwrap.dedent("""\ + [style] + based_on_style = yapf + use_tabs = true + indent_width = 1 + """) + with utils.TempFileContents(self.test_tmpdir, style_contents) as stylepath: + self.assertYapfReformats( + unformatted_code, + expected_formatted_code, + extra_options=['--style={0}'.format(stylepath)]) + + def testUseTabsContinuationAlignStyleFixed(self): + unformatted_code = """\ +def foo_function(arg1, arg2, arg3): + return ['hello', 'world',] +""" + expected_formatted_code = """\ +def foo_function( + arg1, arg2, arg3): + return [ + 'hello', + 'world', + ] +""" # noqa: W191,E101 + style_contents = textwrap.dedent("""\ + [style] + based_on_style = yapf + use_tabs = true + column_limit=32 + indent_width=4 + continuation_indent_width=8 + continuation_align_style = fixed + """) + with utils.TempFileContents(self.test_tmpdir, style_contents) as stylepath: + self.assertYapfReformats( + unformatted_code, + expected_formatted_code, + extra_options=['--style={0}'.format(stylepath)]) + + def testUseTabsContinuationAlignStyleVAlignRight(self): + unformatted_code = """\ +def foo_function(arg1, arg2, arg3): + return ['hello', 'world',] +""" + expected_formatted_code = """\ +def foo_function(arg1, arg2, + arg3): + return [ + 'hello', + 'world', + ] +""" # noqa: W191,E101 + style_contents = textwrap.dedent("""\ + [style] + based_on_style = yapf + use_tabs = true + column_limit = 32 + indent_width = 4 + continuation_indent_width = 8 + continuation_align_style = valign-right + """) + with utils.TempFileContents(self.test_tmpdir, style_contents) as stylepath: + self.assertYapfReformats( + unformatted_code, + expected_formatted_code, + extra_options=['--style={0}'.format(stylepath)]) + + def testUseSpacesContinuationAlignStyleFixed(self): + unformatted_code = textwrap.dedent("""\ + def foo_function(arg1, arg2, arg3): + return ['hello', 'world',] + """) + expected_formatted_code = textwrap.dedent("""\ + def foo_function( + arg1, arg2, arg3): + return [ + 'hello', + 'world', + ] + """) + style_contents = textwrap.dedent("""\ + [style] + based_on_style = yapf + column_limit = 32 + indent_width = 4 + continuation_indent_width = 8 + continuation_align_style = fixed + """) + with utils.TempFileContents(self.test_tmpdir, style_contents) as stylepath: + self.assertYapfReformats( + unformatted_code, + expected_formatted_code, + extra_options=['--style={0}'.format(stylepath)]) + + def testUseSpacesContinuationAlignStyleVAlignRight(self): + unformatted_code = textwrap.dedent("""\ + def foo_function(arg1, arg2, arg3): + return ['hello', 'world',] + """) + expected_formatted_code = textwrap.dedent("""\ + def foo_function(arg1, arg2, + arg3): + return [ + 'hello', + 'world', + ] + """) + style_contents = textwrap.dedent("""\ + [style] + based_on_style = yapf + column_limit = 32 + indent_width = 4 + continuation_indent_width = 8 + continuation_align_style = valign-right + """) + with utils.TempFileContents(self.test_tmpdir, style_contents) as stylepath: + self.assertYapfReformats( + unformatted_code, + expected_formatted_code, + extra_options=['--style={0}'.format(stylepath)]) + + def testStyleOutputRoundTrip(self): + unformatted_code = textwrap.dedent("""\ def foo_function(): - if True: - pass - """) - with tempfile.NamedTemporaryFile(dir=self.test_tmpdir, mode='w') as f: - f.write( - textwrap.dedent('''\ - [style] - based_on_style = chromium - USE_TABS = true - INDENT_WIDTH=1 - ''')) - f.flush() + pass + """) + expected_formatted_code = textwrap.dedent("""\ + def foo_function(): + pass + """) + + with utils.NamedTempFile(dirname=self.test_tmpdir) as (stylefile, + stylepath): + p = subprocess.Popen( + YAPF_BINARY + ['--style-help'], + stdout=stylefile, + stdin=subprocess.PIPE, + stderr=subprocess.PIPE) + _, stderrdata = p.communicate() + self.assertEqual(stderrdata, b'') self.assertYapfReformats( unformatted_code, expected_formatted_code, - extra_options=['--style={0}'.format(f.name)]) + extra_options=['--style={0}'.format(stylepath)]) + + def testSpacingBeforeComments(self): + unformatted_code = textwrap.dedent("""\ + A = 42 -class BadInputTest(unittest.TestCase): + # A comment + def x(): + pass + def _(): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + A = 42 + + + # A comment + def x(): + pass + def _(): + pass + """) + self.assertYapfReformats( + unformatted_code, + expected_formatted_code, + extra_options=['--lines', '1-2']) + + def testSpacingBeforeCommentsInDicts(self): + unformatted_code = textwrap.dedent("""\ + A=42 + + X = { + # 'Valid' statuses. + PASSED: # Passed + 'PASSED', + FAILED: # Failed + 'FAILED', + TIMED_OUT: # Timed out. + 'FAILED', + BORKED: # Broken. + 'BROKEN' + } + """) + expected_formatted_code = textwrap.dedent("""\ + A = 42 + + X = { + # 'Valid' statuses. + PASSED: # Passed + 'PASSED', + FAILED: # Failed + 'FAILED', + TIMED_OUT: # Timed out. + 'FAILED', + BORKED: # Broken. + 'BROKEN' + } + """) + self.assertYapfReformats( + unformatted_code, + expected_formatted_code, + extra_options=['--style', 'yapf', '--lines', '1-1']) + + def testDisableWithLinesOption(self): + unformatted_code = textwrap.dedent("""\ + # yapf_lines_bug.py + # yapf: disable + def outer_func(): + def inner_func(): + return + return + # yapf: enable + """) + expected_formatted_code = textwrap.dedent("""\ + # yapf_lines_bug.py + # yapf: disable + def outer_func(): + def inner_func(): + return + return + # yapf: enable + """) + self.assertYapfReformats( + unformatted_code, + expected_formatted_code, + extra_options=['--lines', '1-8']) + + def testDisableWithLineRanges(self): + unformatted_code = textwrap.dedent("""\ + # yapf: disable + a = [ + 1, + 2, + + 3 + ] + """) + expected_formatted_code = textwrap.dedent("""\ + # yapf: disable + a = [ + 1, + 2, + + 3 + ] + """) + self.assertYapfReformats( + unformatted_code, + expected_formatted_code, + extra_options=['--style', 'yapf', '--lines', '1-100']) + + +class BadInputTest(yapf_test_helper.YAPFTest): """Test yapf's behaviour when passed bad input.""" def testBadSyntax(self): code = ' a = 1\n' - self.assertRaises(SyntaxError, yapf_api.FormatCode, code) + self.assertRaises(errors.YapfError, yapf_api.FormatCode, code) + + def testBadCode(self): + code = 'x = """hello\n' + self.assertRaises(errors.YapfError, yapf_api.FormatCode, code) + + +class DiffIndentTest(yapf_test_helper.YAPFTest): + + @staticmethod + def _OwnStyle(): + my_style = style.CreatePEP8Style() + my_style['INDENT_WIDTH'] = 3 + my_style['CONTINUATION_INDENT_WIDTH'] = 3 + return my_style + + def _Check(self, unformatted_code, expected_formatted_code): + formatted_code, _ = yapf_api.FormatCode( + unformatted_code, style_config=style.SetGlobalStyle(self._OwnStyle())) + self.assertEqual(expected_formatted_code, formatted_code) + + def testSimple(self): + unformatted_code = textwrap.dedent("""\ + for i in range(5): + print('bar') + """) + expected_formatted_code = textwrap.dedent("""\ + for i in range(5): + print('bar') + """) + self._Check(unformatted_code, expected_formatted_code) + + +class HorizontallyAlignedTrailingCommentsTest(yapf_test_helper.YAPFTest): + + @staticmethod + def _OwnStyle(): + my_style = style.CreatePEP8Style() + my_style['SPACES_BEFORE_COMMENT'] = [ + 15, + 25, + 35, + ] + return my_style + + def _Check(self, unformatted_code, expected_formatted_code): + formatted_code, _ = yapf_api.FormatCode( + unformatted_code, style_config=style.SetGlobalStyle(self._OwnStyle())) + self.assertCodeEqual(expected_formatted_code, formatted_code) + + def testSimple(self): + unformatted_code = textwrap.dedent("""\ + foo = '1' # Aligned at first list value + + foo = '2__<15>' # Aligned at second list value + + foo = '3____________<25>' # Aligned at third list value + + foo = '4______________________<35>' # Aligned beyond list values + """) + expected_formatted_code = textwrap.dedent("""\ + foo = '1' # Aligned at first list value + + foo = '2__<15>' # Aligned at second list value + + foo = '3____________<25>' # Aligned at third list value + + foo = '4______________________<35>' # Aligned beyond list values + """) + self._Check(unformatted_code, expected_formatted_code) + + def testBlock(self): + unformatted_code = textwrap.dedent("""\ + func(1) # Line 1 + func(2) # Line 2 + # Line 3 + func(3) # Line 4 + # Line 5 + # Line 6 + """) + expected_formatted_code = textwrap.dedent("""\ + func(1) # Line 1 + func(2) # Line 2 + # Line 3 + func(3) # Line 4 + # Line 5 + # Line 6 + """) + self._Check(unformatted_code, expected_formatted_code) + + def testBlockWithLongLine(self): + unformatted_code = textwrap.dedent("""\ + func(1) # Line 1 + func___________________(2) # Line 2 + # Line 3 + func(3) # Line 4 + # Line 5 + # Line 6 + """) + expected_formatted_code = textwrap.dedent("""\ + func(1) # Line 1 + func___________________(2) # Line 2 + # Line 3 + func(3) # Line 4 + # Line 5 + # Line 6 + """) + self._Check(unformatted_code, expected_formatted_code) + + def testBlockFuncSuffix(self): + unformatted_code = textwrap.dedent("""\ + func(1) # Line 1 + func(2) # Line 2 + # Line 3 + func(3) # Line 4 + # Line 5 + # Line 6 + + def Func(): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + func(1) # Line 1 + func(2) # Line 2 + # Line 3 + func(3) # Line 4 + # Line 5 + # Line 6 + + + def Func(): + pass + """) + self._Check(unformatted_code, expected_formatted_code) + + def testBlockCommentSuffix(self): + unformatted_code = textwrap.dedent("""\ + func(1) # Line 1 + func(2) # Line 2 + # Line 3 + func(3) # Line 4 + # Line 5 - SpliceComments makes this part of the previous block + # Line 6 + + # Aligned with prev comment block + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + func(1) # Line 1 + func(2) # Line 2 + # Line 3 + func(3) # Line 4 + # Line 5 - SpliceComments makes this part of the previous block + # Line 6 + + # Aligned with prev comment block + """) # noqa + self._Check(unformatted_code, expected_formatted_code) + + def testBlockIndentedFuncSuffix(self): + unformatted_code = textwrap.dedent("""\ + if True: + func(1) # Line 1 + func(2) # Line 2 + # Line 3 + func(3) # Line 4 + # Line 5 - SpliceComments makes this a new block + # Line 6 + + # Aligned with Func + + def Func(): + pass + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + if True: + func(1) # Line 1 + func(2) # Line 2 + # Line 3 + func(3) # Line 4 + + # Line 5 - SpliceComments makes this a new block + # Line 6 + + # Aligned with Func + + + def Func(): + pass + """) + self._Check(unformatted_code, expected_formatted_code) + + def testBlockIndentedCommentSuffix(self): + unformatted_code = textwrap.dedent("""\ + if True: + func(1) # Line 1 + func(2) # Line 2 + # Line 3 + func(3) # Line 4 + # Line 5 + # Line 6 + + # Not aligned + """) + expected_formatted_code = textwrap.dedent("""\ + if True: + func(1) # Line 1 + func(2) # Line 2 + # Line 3 + func(3) # Line 4 + # Line 5 + # Line 6 + + # Not aligned + """) + self._Check(unformatted_code, expected_formatted_code) + + def testBlockMultiIndented(self): + unformatted_code = textwrap.dedent("""\ + if True: + if True: + if True: + func(1) # Line 1 + func(2) # Line 2 + # Line 3 + func(3) # Line 4 + # Line 5 + # Line 6 + + # Not aligned + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + if True: + if True: + if True: + func(1) # Line 1 + func(2) # Line 2 + # Line 3 + func(3) # Line 4 + # Line 5 + # Line 6 + + # Not aligned + """) + self._Check(unformatted_code, expected_formatted_code) + + def testArgs(self): + unformatted_code = textwrap.dedent("""\ + def MyFunc( + arg1, # Desc 1 + arg2, # Desc 2 + a_longer_var_name, # Desc 3 + arg4, + arg5, # Desc 5 + arg6, + ): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + def MyFunc( + arg1, # Desc 1 + arg2, # Desc 2 + a_longer_var_name, # Desc 3 + arg4, + arg5, # Desc 5 + arg6, + ): + pass + """) + self._Check(unformatted_code, expected_formatted_code) + + def testDisableBlock(self): + unformatted_code = textwrap.dedent("""\ + a() # comment 1 + b() # comment 2 + + # yapf: disable + c() # comment 3 + d() # comment 4 + # yapf: enable + + e() # comment 5 + f() # comment 6 + """) + expected_formatted_code = textwrap.dedent("""\ + a() # comment 1 + b() # comment 2 + + # yapf: disable + c() # comment 3 + d() # comment 4 + # yapf: enable + + e() # comment 5 + f() # comment 6 + """) + self._Check(unformatted_code, expected_formatted_code) + + def testDisabledLine(self): + unformatted_code = textwrap.dedent("""\ + short # comment 1 + do_not_touch1 # yapf: disable + do_not_touch2 # yapf: disable + a_longer_statement # comment 2 + """) + expected_formatted_code = textwrap.dedent("""\ + short # comment 1 + do_not_touch1 # yapf: disable + do_not_touch2 # yapf: disable + a_longer_statement # comment 2 + """) + self._Check(unformatted_code, expected_formatted_code) + + +class _SpacesAroundDictListTupleTestImpl(yapf_test_helper.YAPFTest): + + @staticmethod + def _OwnStyle(): + my_style = style.CreatePEP8Style() + my_style['DISABLE_ENDING_COMMA_HEURISTIC'] = True + my_style['SPLIT_ALL_COMMA_SEPARATED_VALUES'] = False + my_style['SPLIT_ARGUMENTS_WHEN_COMMA_TERMINATED'] = False + return my_style + + def _Check(self, unformatted_code, expected_formatted_code): + formatted_code, _ = yapf_api.FormatCode( + unformatted_code, style_config=style.SetGlobalStyle(self._OwnStyle())) + self.assertEqual(expected_formatted_code, formatted_code) + + def setUp(self): + self.maxDiff = None + + +class SpacesAroundDictTest(_SpacesAroundDictListTupleTestImpl): + + @classmethod + def _OwnStyle(cls): + style = super(SpacesAroundDictTest, cls)._OwnStyle() + style['SPACES_AROUND_DICT_DELIMITERS'] = True + + return style + + def testStandard(self): + unformatted_code = textwrap.dedent("""\ + {1 : 2} + {k:v for k, v in other.items()} + {k for k in [1, 2, 3]} + + # The following statements should not change + {} + {1 : 2} # yapf: disable + + # yapf: disable + {1 : 2} + # yapf: enable + + # Dict settings should not impact lists or tuples + [1, 2] + (3, 4) + """) + expected_formatted_code = textwrap.dedent("""\ + { 1: 2 } + { k: v for k, v in other.items() } + { k for k in [1, 2, 3] } + + # The following statements should not change + {} + {1 : 2} # yapf: disable + + # yapf: disable + {1 : 2} + # yapf: enable + + # Dict settings should not impact lists or tuples + [1, 2] + (3, 4) + """) + self._Check(unformatted_code, expected_formatted_code) + + +class SpacesAroundListTest(_SpacesAroundDictListTupleTestImpl): + + @classmethod + def _OwnStyle(cls): + style = super(SpacesAroundListTest, cls)._OwnStyle() + style['SPACES_AROUND_LIST_DELIMITERS'] = True + + return style + + def testStandard(self): + unformatted_code = textwrap.dedent("""\ + [a,b,c] + [4,5,] + [6, [7, 8], 9] + [v for v in [1,2,3] if v & 1] + + # The following statements should not change + index[0] + index[a, b] + [] + [v for v in [1,2,3] if v & 1] # yapf: disable + + # yapf: disable + [a,b,c] + [4,5,] + # yapf: enable + + # List settings should not impact dicts or tuples + {a: b} + (1, 2) + """) + expected_formatted_code = textwrap.dedent("""\ + [ a, b, c ] + [ 4, 5, ] + [ 6, [ 7, 8 ], 9 ] + [ v for v in [ 1, 2, 3 ] if v & 1 ] + + # The following statements should not change + index[0] + index[a, b] + [] + [v for v in [1,2,3] if v & 1] # yapf: disable + + # yapf: disable + [a,b,c] + [4,5,] + # yapf: enable + + # List settings should not impact dicts or tuples + {a: b} + (1, 2) + """) + self._Check(unformatted_code, expected_formatted_code) + + +class SpacesAroundTupleTest(_SpacesAroundDictListTupleTestImpl): + + @classmethod + def _OwnStyle(cls): + style = super(SpacesAroundTupleTest, cls)._OwnStyle() + style['SPACES_AROUND_TUPLE_DELIMITERS'] = True + + return style + + def testStandard(self): + unformatted_code = textwrap.dedent("""\ + (0, 1) + (2, 3) + (4, 5, 6,) + func((7, 8), 9) + + # The following statements should not change + func(1, 2) + (this_func or that_func)(3, 4) + if (True and False): pass + () + + (0, 1) # yapf: disable + + # yapf: disable + (0, 1) + (2, 3) + # yapf: enable + + # Tuple settings should not impact dicts or lists + {a: b} + [3, 4] + """) + expected_formatted_code = textwrap.dedent("""\ + ( 0, 1 ) + ( 2, 3 ) + ( 4, 5, 6, ) + func(( 7, 8 ), 9) + + # The following statements should not change + func(1, 2) + (this_func or that_func)(3, 4) + if (True and False): pass + () + + (0, 1) # yapf: disable + + # yapf: disable + (0, 1) + (2, 3) + # yapf: enable + + # Tuple settings should not impact dicts or lists + {a: b} + [3, 4] + """) + self._Check(unformatted_code, expected_formatted_code) if __name__ == '__main__': diff --git a/yapftests/yapf_test_helper.py b/yapftests/yapf_test_helper.py new file mode 100644 index 000000000..61aa2c51b --- /dev/null +++ b/yapftests/yapf_test_helper.py @@ -0,0 +1,92 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Support module for tests for yapf.""" + +import difflib +import sys +import unittest + +from yapf.pytree import blank_line_calculator +from yapf.pytree import comment_splicer +from yapf.pytree import continuation_splicer +from yapf.pytree import pytree_unwrapper +from yapf.pytree import pytree_utils +from yapf.pytree import pytree_visitor +from yapf.pytree import split_penalty +from yapf.pytree import subtype_assigner +from yapf.yapflib import identify_container +from yapf.yapflib import style + + +class YAPFTest(unittest.TestCase): + + def __init__(self, *args): + super(YAPFTest, self).__init__(*args) + + def assertCodeEqual(self, expected_code, code): + if code != expected_code: + msg = ['Code format mismatch:', 'Expected:'] + linelen = style.Get('COLUMN_LIMIT') + for line in expected_code.splitlines(): + if len(line) > linelen: + msg.append('!> %s' % line) + else: + msg.append(' > %s' % line) + msg.append('Actual:') + for line in code.splitlines(): + if len(line) > linelen: + msg.append('!> %s' % line) + else: + msg.append(' > %s' % line) + msg.append('Diff:') + msg.extend( + difflib.unified_diff( + code.splitlines(), + expected_code.splitlines(), + fromfile='actual', + tofile='expected', + lineterm='')) + self.fail('\n'.join(msg)) + + +def ParseAndUnwrap(code, dumptree=False): + """Produces logical lines from the given code. + + Parses the code into a tree, performs comment splicing and runs the + unwrapper. + + Arguments: + code: code to parse as a string + dumptree: if True, the parsed pytree (after comment splicing) is dumped + to stderr. Useful for debugging. + + Returns: + List of logical lines. + """ + tree = pytree_utils.ParseCodeToTree(code) + comment_splicer.SpliceComments(tree) + continuation_splicer.SpliceContinuations(tree) + subtype_assigner.AssignSubtypes(tree) + identify_container.IdentifyContainers(tree) + split_penalty.ComputeSplitPenalties(tree) + blank_line_calculator.CalculateBlankLines(tree) + + if dumptree: + pytree_visitor.DumpPyTree(tree, target_stream=sys.stderr) + + llines = pytree_unwrapper.UnwrapPyTree(tree) + for lline in llines: + lline.CalculateFormattingInformation() + + return llines