diff --git a/Cargo.toml b/Cargo.toml index 67ef0af27c..228c0b9989 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "hp" version = "0.1.0" -edition = "2021" +edition = "2024" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [lib] @@ -10,7 +10,7 @@ crate-type = ["cdylib"] [dependencies] openssl-sys = { version = "*", features = ["vendored"] } -pyo3 = { version = "*", features = ["extension-module"] } +pyo3 = { version = ">= 0.28.2", features = ["extension-module"] } rust-htslib = "*" rayon = "*" tokio = "*" diff --git a/pixi.lock b/pixi.lock index 2c515f4591..7b898cfe85 100644 --- a/pixi.lock +++ b/pixi.lock @@ -132,7 +132,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/2c/02/3623e6169bed617ed1e2d372f7c69f92ec28d54c4dfc997055c8578ec148/virtualenv-21.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/32/8e/b94ec25fcd4384de2a1f5e0403284f005ec743c04bafdb6698c851df5c6d/tuspy-1.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/37/aa/111610d8bf5b1bb7a295a048fc648cec346347a8b0be5881defd2d1b4a52/oyaml-1.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/3a/7a/882d99539b19b1490cac5d77c67338d126e4122c8276bf640e411650c830/twine-6.2.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl @@ -141,7 +140,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/3f/51/d4db610ef29373b879047326cbf6fa98b6c1969d6f6dc423279de2b1be2c/requests_toolbelt-1.0.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/42/77/de194443bf38daed9452139e960c632b0ef9f9a5dd9ce605fdf18ca9f1b1/id-1.6.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/43/e3/fdc657359e919462369869f1c9f0e973f353f9a9ee295a39b1fea8ee1a77/pillow-12.2.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/48/ca/36339329c4604adbcc99c899b7eb1ce1a555c499b6a6860757dc9bfed36d/narwhals-2.22.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/4b/f8/d0118a2f5f23b02cd166fa385c60f9b0d4f9194f574e2b31cef350ad7223/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/53/78/ddd0affa881b1e09bc59f11ab829a7904f4bca29d25c34cbb9395e8e0061/allure_python_commons-2.16.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl @@ -152,12 +150,10 @@ environments: - pypi: https://files.pythonhosted.org/packages/5f/fc/a7bf5b6e4e617b45f90f2d9d2a68519c249c81dd4fc2658c7a2a61c4f4b7/aiohappyeyeballs-2.6.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/65/b6/09b01cdbc15224e2850365192d17b7bdebb8bdbd8780ed221fcdf0d9a515/pandas-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/6a/bd/d91c5e39f490a49df14320f4e8c80161cfcce09f1e2cde1edd16a551abb3/frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl - pypi: https://files.pythonhosted.org/packages/6a/fd/6adabd4f6d5e686f97dd02ce7fce3fe4cf672cbac36b8f67ff4040e8ad8b/msgpack-1.2.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/6b/67/6ea73b2173bcb1abe3a947fbd090cf96ee1b0a75f3f690e8902dc8ffdbfa/py2bit-1.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/6b/b2/d17b2722c636d64b4e77ddc68d8d0625719d39f94021be8719a218af4c0a/backports_zstd-1.6.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/6c/c2/63fdda36c56437eeb44aaf9493c8bcd62ce230ab1598924fc626ffbfa943/scikit_learn-1.9.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/74/f5/9373290775639cb67a2fce7f629a1c240dce9f12fe927bc32b2736e16dfc/argcomplete-3.6.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/77/c7/2342da9830e3e9d4870305ca5d2091d2a83284f2953079b7bdd3b5e029d8/fonttools-4.63.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/78/17/853354204e1ca022d6b7d011ca7f3206c4f8faa3cc743e92609b49c1d83f/tinydb-4.8.2-py3-none-any.whl @@ -166,7 +162,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7a/da/323a01c349bd5fb01bb6652e314d9bb218cee630a736bdb810ad50e4013f/yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/7a/f6/51d8a97116de23c9280c1fa3b813bc088f8571ce5936ba84af1ecf13ed45/pybigwig-0.3.25-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7e/32/a7125fb28c4261a627f999d5fb4afff25b523800faed2c30979949d6facd/pydot-4.0.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7f/66/b15ce62552d84bbfcec9a4873ab79d993a1dd4edb922cbfccae192bd5b5f/jaraco.classes-3.4.0-py3-none-any.whl @@ -307,13 +302,11 @@ environments: - pypi: https://files.pythonhosted.org/packages/1f/99/d1c90d6041656cc6ee229dc99cd67fd0cd5aec3c5f7d72fffc27cc750054/cryptography-49.0.0.tar.gz - pypi: https://files.pythonhosted.org/packages/22/00/ff3009c88e65de8011630acf8ab5a09cb2becd2aaf47fba2f3449f6224e9/lxml-6.1.1-cp312-cp312-macosx_10_13_x86_64.whl - pypi: https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/24/f1/392f8c5bfc16f66a0d2d41561c01627c228fe7ed2a0d056ef11315042570/pandas-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl - pypi: https://files.pythonhosted.org/packages/27/1a/1f68f9ba0c207934b35b86a8ca3aad8395a3d6dd7921c0686e23853ff5a9/mccabe-0.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/7f/a946aa4f8752b37102b41e64dca18a1976ac705c3a0d1dfe74d820a02552/mistune-3.2.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2c/02/3623e6169bed617ed1e2d372f7c69f92ec28d54c4dfc997055c8578ec148/virtualenv-21.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/32/8e/b94ec25fcd4384de2a1f5e0403284f005ec743c04bafdb6698c851df5c6d/tuspy-1.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/37/aa/111610d8bf5b1bb7a295a048fc648cec346347a8b0be5881defd2d1b4a52/oyaml-1.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/3a/7a/882d99539b19b1490cac5d77c67338d126e4122c8276bf640e411650c830/twine-6.2.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/3c/6b/72e5f22f3c5069bd950f107110cd17468cdb43abda3c55b48c4013e7b667/bioblend-1.9.0-py3-none-any.whl @@ -322,7 +315,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/42/77/de194443bf38daed9452139e960c632b0ef9f9a5dd9ce605fdf18ca9f1b1/id-1.6.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/44/a0/c815bea63117fa63e4e1c01f8a1110d2112fa003f838e6467094ec2432ce/fonttools-4.63.0-cp312-cp312-macosx_10_13_x86_64.whl - pypi: https://files.pythonhosted.org/packages/46/3b/9dedd2e35cebcd4eb3539514b5ac674fd45654404839e6f9eff6e25d67c7/py2bit-1.0.1.tar.gz - - pypi: https://files.pythonhosted.org/packages/48/ca/36339329c4604adbcc99c899b7eb1ce1a555c499b6a6860757dc9bfed36d/narwhals-2.22.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/53/78/ddd0affa881b1e09bc59f11ab829a7904f4bca29d25c34cbb9395e8e0061/allure_python_commons-2.16.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/58/9d/40b6267367182187139a4000b82a3b287d84d745bccd808e75d916920e9d/bleach-6.4.0-py3-none-any.whl @@ -337,7 +329,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/78/17/853354204e1ca022d6b7d011ca7f3206c4f8faa3cc743e92609b49c1d83f/tinydb-4.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7e/32/a7125fb28c4261a627f999d5fb4afff25b523800faed2c30979949d6facd/pydot-4.0.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7f/66/b15ce62552d84bbfcec9a4873ab79d993a1dd4edb922cbfccae192bd5b5f/jaraco.classes-3.4.0-py3-none-any.whl @@ -361,7 +352,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ac/1f/d10d6ad23c86c62d90d867d0506881a392ec6ef06885b858eaab868dd356/deeptoolsintervals-0.1.9.tar.gz - - pypi: https://files.pythonhosted.org/packages/ac/20/75f915ff375d6249e6550ac740fdbbd66159a068fd3af1400ff62036b07a/scikit_learn-1.9.0-cp312-cp312-macosx_10_13_x86_64.whl - pypi: https://files.pythonhosted.org/packages/ac/3e/61d11b779e4e83ee8661439af070f6f138060cd41f0346a0a70432f9e55c/cwl_upgrader-1.2.15-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl @@ -503,17 +493,14 @@ environments: - pypi: https://files.pythonhosted.org/packages/2c/02/3623e6169bed617ed1e2d372f7c69f92ec28d54c4dfc997055c8578ec148/virtualenv-21.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/32/8e/b94ec25fcd4384de2a1f5e0403284f005ec743c04bafdb6698c851df5c6d/tuspy-1.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/37/aa/111610d8bf5b1bb7a295a048fc648cec346347a8b0be5881defd2d1b4a52/oyaml-1.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/3a/7a/882d99539b19b1490cac5d77c67338d126e4122c8276bf640e411650c830/twine-6.2.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/3b/00/2344469e2084fb287c2e0b57b72910309874c3245463acd6cf5e3db69324/appdirs-1.4.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/3c/6b/72e5f22f3c5069bd950f107110cd17468cdb43abda3c55b48c4013e7b667/bioblend-1.9.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/3c/a7/552a7821597c632b907f7bfe8f36f9f572777af8ef8a48353041cf8e091a/scikit_learn-1.9.0-cp314-cp314-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/3d/6f/221654a39431edc7e9685702ab06fbb22bb313629e0187c0ed4462ba8af7/schema_salad-8.9.20260417192335-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/3f/51/d4db610ef29373b879047326cbf6fa98b6c1969d6f6dc423279de2b1be2c/requests_toolbelt-1.0.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/42/77/de194443bf38daed9452139e960c632b0ef9f9a5dd9ce605fdf18ca9f1b1/id-1.6.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/46/3b/9dedd2e35cebcd4eb3539514b5ac674fd45654404839e6f9eff6e25d67c7/py2bit-1.0.1.tar.gz - - pypi: https://files.pythonhosted.org/packages/48/ca/36339329c4604adbcc99c899b7eb1ce1a555c499b6a6860757dc9bfed36d/narwhals-2.22.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/49/b2/97980f3ad4fae37dd7fe31626e2bf75fbf8bdf5d303950ec1fab39a12da8/kiwisolver-1.5.0-cp314-cp314-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/53/78/ddd0affa881b1e09bc59f11ab829a7904f4bca29d25c34cbb9395e8e0061/allure_python_commons-2.16.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl @@ -525,14 +512,12 @@ environments: - pypi: https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/63/b1/4260d67d6bd85e58a66b72d54ce15d5de789b6f3870cc6bedf8ff9667401/propcache-0.5.2-cp314-cp314-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/68/10/bf2d6738d72748b961a3751ab89522d58c54efc36a8e1a12161216cd45cf/pandas-3.0.3-cp314-cp314-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/6a/b7/9366ed44ced9b7ef357ab48c94205280276db9d7f064aa3012a97227e966/h5py-3.16.0-cp314-cp314-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/72/65/a4559d4ff4a210d2a26d7ca0712e4e738cbfc963135f86e8ae807ca420eb/pysam-0.24.0-cp314-cp314-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/74/f5/9373290775639cb67a2fce7f629a1c240dce9f12fe927bc32b2736e16dfc/argcomplete-3.6.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/17/853354204e1ca022d6b7d011ca7f3206c4f8faa3cc743e92609b49c1d83f/tinydb-4.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7c/03/b8cdb625a21f710dfa11bbca1f48fb4057d2c0286975f8b415bf80942c99/matplotlib-3.11.0-cp314-cp314-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/7e/32/a7125fb28c4261a627f999d5fb4afff25b523800faed2c30979949d6facd/pydot-4.0.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl @@ -3579,8 +3564,6 @@ packages: - pysam>=0.23 - pybigwig>=0.3 - py2bit>=0.3 - - pandas>=2.2 - - scikit-learn>=1.6 - deeptoolsintervals>=0.1 - maturin - flake8 ; extra == 'actions' @@ -3855,96 +3838,6 @@ packages: - setuptools ; extra == 'dev' - xmlschema ; extra == 'dev' requires_python: '>=3.10' -- pypi: https://files.pythonhosted.org/packages/24/f1/392f8c5bfc16f66a0d2d41561c01627c228fe7ed2a0d056ef11315042570/pandas-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl - name: pandas - version: 3.0.3 - sha256: fed2ff7fd9779120e388e285fc029bd5cf9490cdd2e4166a9ee22c0e49a9ab09 - requires_dist: - - numpy>=1.26.0 ; python_full_version < '3.14' - - numpy>=2.3.3 ; python_full_version >= '3.14' - - python-dateutil>=2.8.2 - - tzdata ; sys_platform == 'win32' - - tzdata ; sys_platform == 'emscripten' - - hypothesis>=6.116.0 ; extra == 'test' - - pytest>=8.3.4 ; extra == 'test' - - pytest-xdist>=3.6.1 ; extra == 'test' - - pyarrow>=13.0.0 ; extra == 'pyarrow' - - bottleneck>=1.4.2 ; extra == 'performance' - - numba>=0.60.0 ; extra == 'performance' - - numexpr>=2.10.2 ; extra == 'performance' - - scipy>=1.14.1 ; extra == 'computation' - - xarray>=2024.10.0 ; extra == 'computation' - - fsspec>=2024.10.0 ; extra == 'fss' - - s3fs>=2024.10.0 ; extra == 'aws' - - gcsfs>=2024.10.0 ; extra == 'gcp' - - odfpy>=1.4.1 ; extra == 'excel' - - openpyxl>=3.1.5 ; extra == 'excel' - - python-calamine>=0.3.0 ; extra == 'excel' - - pyxlsb>=1.0.10 ; extra == 'excel' - - xlrd>=2.0.1 ; extra == 'excel' - - xlsxwriter>=3.2.0 ; extra == 'excel' - - pyarrow>=13.0.0 ; extra == 'parquet' - - pyarrow>=13.0.0 ; extra == 'feather' - - pyiceberg>=0.8.1 ; extra == 'iceberg' - - tables>=3.10.1 ; extra == 'hdf5' - - pyreadstat>=1.2.8 ; extra == 'spss' - - sqlalchemy>=2.0.36 ; extra == 'postgresql' - - psycopg2>=2.9.10 ; extra == 'postgresql' - - adbc-driver-postgresql>=1.2.0 ; extra == 'postgresql' - - sqlalchemy>=2.0.36 ; extra == 'mysql' - - pymysql>=1.1.1 ; extra == 'mysql' - - sqlalchemy>=2.0.36 ; extra == 'sql-other' - - adbc-driver-postgresql>=1.2.0 ; extra == 'sql-other' - - adbc-driver-sqlite>=1.2.0 ; extra == 'sql-other' - - beautifulsoup4>=4.12.3 ; extra == 'html' - - html5lib>=1.1 ; extra == 'html' - - lxml>=5.3.0 ; extra == 'html' - - lxml>=5.3.0 ; extra == 'xml' - - matplotlib>=3.9.3 ; extra == 'plot' - - jinja2>=3.1.5 ; extra == 'output-formatting' - - tabulate>=0.9.0 ; extra == 'output-formatting' - - pyqt5>=5.15.9 ; extra == 'clipboard' - - qtpy>=2.4.2 ; extra == 'clipboard' - - zstandard>=0.23.0 ; extra == 'compression' - - pytz>=2020.1 ; extra == 'timezone' - - adbc-driver-postgresql>=1.2.0 ; extra == 'all' - - adbc-driver-sqlite>=1.2.0 ; extra == 'all' - - beautifulsoup4>=4.12.3 ; extra == 'all' - - bottleneck>=1.4.2 ; extra == 'all' - - fastparquet>=2024.11.0 ; extra == 'all' - - fsspec>=2024.10.0 ; extra == 'all' - - gcsfs>=2024.10.0 ; extra == 'all' - - html5lib>=1.1 ; extra == 'all' - - hypothesis>=6.116.0 ; extra == 'all' - - jinja2>=3.1.5 ; extra == 'all' - - lxml>=5.3.0 ; extra == 'all' - - matplotlib>=3.9.3 ; extra == 'all' - - numba>=0.60.0 ; extra == 'all' - - numexpr>=2.10.2 ; extra == 'all' - - odfpy>=1.4.1 ; extra == 'all' - - openpyxl>=3.1.5 ; extra == 'all' - - psycopg2>=2.9.10 ; extra == 'all' - - pyarrow>=13.0.0 ; extra == 'all' - - pyiceberg>=0.8.1 ; extra == 'all' - - pymysql>=1.1.1 ; extra == 'all' - - pyqt5>=5.15.9 ; extra == 'all' - - pyreadstat>=1.2.8 ; extra == 'all' - - pytest>=8.3.4 ; extra == 'all' - - pytest-xdist>=3.6.1 ; extra == 'all' - - python-calamine>=0.3.0 ; extra == 'all' - - pytz>=2020.1 ; extra == 'all' - - pyxlsb>=1.0.10 ; extra == 'all' - - qtpy>=2.4.2 ; extra == 'all' - - scipy>=1.14.1 ; extra == 'all' - - s3fs>=2024.10.0 ; extra == 'all' - - sqlalchemy>=2.0.36 ; extra == 'all' - - tables>=3.10.1 ; extra == 'all' - - tabulate>=0.9.0 ; extra == 'all' - - xarray>=2024.10.0 ; extra == 'all' - - xlrd>=2.0.1 ; extra == 'all' - - xlsxwriter>=3.2.0 ; extra == 'all' - - zstandard>=0.23.0 ; extra == 'all' - requires_python: '>=3.11' - pypi: https://files.pythonhosted.org/packages/27/1a/1f68f9ba0c207934b35b86a8ca3aad8395a3d6dd7921c0686e23853ff5a9/mccabe-0.7.0-py2.py3-none-any.whl name: mccabe version: 0.7.0 @@ -4025,11 +3918,6 @@ packages: - sphinx-autobuild==2021.3.14 ; extra == 'dev' - sphinx==1.7.1 ; extra == 'dev' requires_python: '>=3.5.3' -- pypi: https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl - name: threadpoolctl - version: 3.6.0 - sha256: 43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb - requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/37/aa/111610d8bf5b1bb7a295a048fc648cec346347a8b0be5881defd2d1b4a52/oyaml-1.0-py2.py3-none-any.whl name: oyaml version: '1.0' @@ -4073,70 +3961,6 @@ packages: - tuspy - pytest ; extra == 'testing' requires_python: '>=3.10' -- pypi: https://files.pythonhosted.org/packages/3c/a7/552a7821597c632b907f7bfe8f36f9f572777af8ef8a48353041cf8e091a/scikit_learn-1.9.0-cp314-cp314-macosx_12_0_arm64.whl - name: scikit-learn - version: 1.9.0 - sha256: 24360002ae845e7866522b0a5bbf690802e7bc388cac8663502e78aa98598aa2 - requires_dist: - - numpy>=1.24.1 - - scipy>=1.10.0 - - joblib>=1.4.0 - - narwhals>=2.0.1 - - threadpoolctl>=3.5.0 - - numpy>=1.24.1 ; extra == 'build' - - scipy>=1.10.0 ; extra == 'build' - - cython>=3.1.2 ; extra == 'build' - - meson-python>=0.17.1 ; extra == 'build' - - numpy>=1.24.1 ; extra == 'install' - - scipy>=1.10.0 ; extra == 'install' - - joblib>=1.4.0 ; extra == 'install' - - narwhals>=2.0.1 ; extra == 'install' - - threadpoolctl>=3.5.0 ; extra == 'install' - - matplotlib>=3.6.1 ; extra == 'benchmark' - - pandas>=1.5.0 ; extra == 'benchmark' - - memory-profiler>=0.57.0 ; extra == 'benchmark' - - matplotlib>=3.6.1 ; extra == 'docs' - - scikit-image>=0.22.0 ; extra == 'docs' - - pandas>=1.5.0 ; extra == 'docs' - - rich>=14.1.0 ; extra == 'docs' - - seaborn>=0.13.0 ; extra == 'docs' - - memory-profiler>=0.57.0 ; extra == 'docs' - - sphinx>=7.3.7 ; extra == 'docs' - - sphinx-copybutton>=0.5.2 ; extra == 'docs' - - sphinx-gallery>=0.17.1 ; extra == 'docs' - - numpydoc>=1.2.0 ; extra == 'docs' - - pillow>=12.1.1 ; extra == 'docs' - - pooch>=1.8.0 ; extra == 'docs' - - sphinx-prompt>=1.4.0 ; extra == 'docs' - - sphinxext-opengraph>=0.9.1 ; extra == 'docs' - - plotly>=5.22.0 ; extra == 'docs' - - polars>=0.20.30 ; extra == 'docs' - - sphinx-design>=0.6.0 ; extra == 'docs' - - sphinxcontrib-sass>=0.3.4 ; extra == 'docs' - - pydata-sphinx-theme>=0.15.3 ; extra == 'docs' - - sphinx-remove-toctrees>=1.0.0.post1 ; extra == 'docs' - - towncrier>=24.8.0 ; extra == 'docs' - - matplotlib>=3.6.1 ; extra == 'examples' - - scikit-image>=0.22.0 ; extra == 'examples' - - pandas>=1.5.0 ; extra == 'examples' - - rich>=14.1.0 ; extra == 'examples' - - seaborn>=0.13.0 ; extra == 'examples' - - pooch>=1.8.0 ; extra == 'examples' - - plotly>=5.22.0 ; extra == 'examples' - - matplotlib>=3.6.1 ; extra == 'tests' - - pandas>=1.5.0 ; extra == 'tests' - - rich>=14.1.0 ; extra == 'tests' - - pytest>=7.1.2 ; extra == 'tests' - - pytest-cov>=2.9.0 ; extra == 'tests' - - ruff>=0.12.2 ; extra == 'tests' - - mypy>=1.15 ; extra == 'tests' - - pyamg>=5.0.0 ; extra == 'tests' - - polars>=0.20.30 ; extra == 'tests' - - pyarrow>=13.0.0 ; extra == 'tests' - - numpydoc>=1.2.0 ; extra == 'tests' - - pooch>=1.8.0 ; extra == 'tests' - - conda-lock==3.0.1 ; extra == 'maintenance' - requires_python: '>=3.11' - pypi: https://files.pythonhosted.org/packages/3d/6f/221654a39431edc7e9685702ab06fbb22bb313629e0187c0ed4462ba8af7/schema_salad-8.9.20260417192335-py3-none-any.whl name: schema-salad version: 8.9.20260417192335 @@ -4252,28 +4076,6 @@ packages: name: py2bit version: 1.0.1 sha256: 4972f85eb3844cdfba43eb54ab3c8349a0536e03dfd7db07ca8d3447285ad20c -- pypi: https://files.pythonhosted.org/packages/48/ca/36339329c4604adbcc99c899b7eb1ce1a555c499b6a6860757dc9bfed36d/narwhals-2.22.1-py3-none-any.whl - name: narwhals - version: 2.22.1 - sha256: 60567d774edf77db53906f89d9fbd164e66e56d66d388e1e6990f17ac33cfb53 - requires_dist: - - cudf-cu12>=24.10.0 ; sys_platform == 'linux' and extra == 'cudf' - - dask[dataframe]>=2024.8 ; extra == 'dask' - - duckdb>=1.1 ; extra == 'duckdb' - - ibis-framework>=6.0.0 ; extra == 'ibis' - - rich>=12.4.4 ; extra == 'ibis' - - packaging>=21.3 ; extra == 'ibis' - - pyarrow-hotfix>=0.7 ; extra == 'ibis' - - modin>=0.22.0 ; extra == 'modin' - - pandas>=1.3.4 ; extra == 'pandas' - - polars>=0.20.4 ; extra == 'polars' - - pyarrow>=13.0.0 ; extra == 'pyarrow' - - pyspark>=3.5.0 ; extra == 'pyspark' - - pyspark[connect]>=3.5.0 ; extra == 'pyspark-connect' - - narwhals[duckdb] ; extra == 'sql' - - sqlparse>=0.5.5 ; extra == 'sql' - - sqlframe>=3.22.0,!=3.39.3 ; extra == 'sqlframe' - requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/49/b2/97980f3ad4fae37dd7fe31626e2bf75fbf8bdf5d303950ec1fab39a12da8/kiwisolver-1.5.0-cp314-cp314-macosx_11_0_arm64.whl name: kiwisolver version: 1.5.0 @@ -4410,186 +4212,6 @@ packages: version: 26.1.0 sha256: c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309 requires_python: '>=3.9' -- pypi: https://files.pythonhosted.org/packages/65/b6/09b01cdbc15224e2850365192d17b7bdebb8bdbd8780ed221fcdf0d9a515/pandas-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl - name: pandas - version: 3.0.3 - sha256: 6dc0b3fd2169c9157deed50b4d519553a3655c8c6a96027136d654592be973a9 - requires_dist: - - numpy>=1.26.0 ; python_full_version < '3.14' - - numpy>=2.3.3 ; python_full_version >= '3.14' - - python-dateutil>=2.8.2 - - tzdata ; sys_platform == 'win32' - - tzdata ; sys_platform == 'emscripten' - - hypothesis>=6.116.0 ; extra == 'test' - - pytest>=8.3.4 ; extra == 'test' - - pytest-xdist>=3.6.1 ; extra == 'test' - - pyarrow>=13.0.0 ; extra == 'pyarrow' - - bottleneck>=1.4.2 ; extra == 'performance' - - numba>=0.60.0 ; extra == 'performance' - - numexpr>=2.10.2 ; extra == 'performance' - - scipy>=1.14.1 ; extra == 'computation' - - xarray>=2024.10.0 ; extra == 'computation' - - fsspec>=2024.10.0 ; extra == 'fss' - - s3fs>=2024.10.0 ; extra == 'aws' - - gcsfs>=2024.10.0 ; extra == 'gcp' - - odfpy>=1.4.1 ; extra == 'excel' - - openpyxl>=3.1.5 ; extra == 'excel' - - python-calamine>=0.3.0 ; extra == 'excel' - - pyxlsb>=1.0.10 ; extra == 'excel' - - xlrd>=2.0.1 ; extra == 'excel' - - xlsxwriter>=3.2.0 ; extra == 'excel' - - pyarrow>=13.0.0 ; extra == 'parquet' - - pyarrow>=13.0.0 ; extra == 'feather' - - pyiceberg>=0.8.1 ; extra == 'iceberg' - - tables>=3.10.1 ; extra == 'hdf5' - - pyreadstat>=1.2.8 ; extra == 'spss' - - sqlalchemy>=2.0.36 ; extra == 'postgresql' - - psycopg2>=2.9.10 ; extra == 'postgresql' - - adbc-driver-postgresql>=1.2.0 ; extra == 'postgresql' - - sqlalchemy>=2.0.36 ; extra == 'mysql' - - pymysql>=1.1.1 ; extra == 'mysql' - - sqlalchemy>=2.0.36 ; extra == 'sql-other' - - adbc-driver-postgresql>=1.2.0 ; extra == 'sql-other' - - adbc-driver-sqlite>=1.2.0 ; extra == 'sql-other' - - beautifulsoup4>=4.12.3 ; extra == 'html' - - html5lib>=1.1 ; extra == 'html' - - lxml>=5.3.0 ; extra == 'html' - - lxml>=5.3.0 ; extra == 'xml' - - matplotlib>=3.9.3 ; extra == 'plot' - - jinja2>=3.1.5 ; extra == 'output-formatting' - - tabulate>=0.9.0 ; extra == 'output-formatting' - - pyqt5>=5.15.9 ; extra == 'clipboard' - - qtpy>=2.4.2 ; extra == 'clipboard' - - zstandard>=0.23.0 ; extra == 'compression' - - pytz>=2020.1 ; extra == 'timezone' - - adbc-driver-postgresql>=1.2.0 ; extra == 'all' - - adbc-driver-sqlite>=1.2.0 ; extra == 'all' - - beautifulsoup4>=4.12.3 ; extra == 'all' - - bottleneck>=1.4.2 ; extra == 'all' - - fastparquet>=2024.11.0 ; extra == 'all' - - fsspec>=2024.10.0 ; extra == 'all' - - gcsfs>=2024.10.0 ; extra == 'all' - - html5lib>=1.1 ; extra == 'all' - - hypothesis>=6.116.0 ; extra == 'all' - - jinja2>=3.1.5 ; extra == 'all' - - lxml>=5.3.0 ; extra == 'all' - - matplotlib>=3.9.3 ; extra == 'all' - - numba>=0.60.0 ; extra == 'all' - - numexpr>=2.10.2 ; extra == 'all' - - odfpy>=1.4.1 ; extra == 'all' - - openpyxl>=3.1.5 ; extra == 'all' - - psycopg2>=2.9.10 ; extra == 'all' - - pyarrow>=13.0.0 ; extra == 'all' - - pyiceberg>=0.8.1 ; extra == 'all' - - pymysql>=1.1.1 ; extra == 'all' - - pyqt5>=5.15.9 ; extra == 'all' - - pyreadstat>=1.2.8 ; extra == 'all' - - pytest>=8.3.4 ; extra == 'all' - - pytest-xdist>=3.6.1 ; extra == 'all' - - python-calamine>=0.3.0 ; extra == 'all' - - pytz>=2020.1 ; extra == 'all' - - pyxlsb>=1.0.10 ; extra == 'all' - - qtpy>=2.4.2 ; extra == 'all' - - scipy>=1.14.1 ; extra == 'all' - - s3fs>=2024.10.0 ; extra == 'all' - - sqlalchemy>=2.0.36 ; extra == 'all' - - tables>=3.10.1 ; extra == 'all' - - tabulate>=0.9.0 ; extra == 'all' - - xarray>=2024.10.0 ; extra == 'all' - - xlrd>=2.0.1 ; extra == 'all' - - xlsxwriter>=3.2.0 ; extra == 'all' - - zstandard>=0.23.0 ; extra == 'all' - requires_python: '>=3.11' -- pypi: https://files.pythonhosted.org/packages/68/10/bf2d6738d72748b961a3751ab89522d58c54efc36a8e1a12161216cd45cf/pandas-3.0.3-cp314-cp314-macosx_11_0_arm64.whl - name: pandas - version: 3.0.3 - sha256: bab900348131a7db1f69a7309ef141fd5680f1487094193bcbbb61791573bf8f - requires_dist: - - numpy>=1.26.0 ; python_full_version < '3.14' - - numpy>=2.3.3 ; python_full_version >= '3.14' - - python-dateutil>=2.8.2 - - tzdata ; sys_platform == 'win32' - - tzdata ; sys_platform == 'emscripten' - - hypothesis>=6.116.0 ; extra == 'test' - - pytest>=8.3.4 ; extra == 'test' - - pytest-xdist>=3.6.1 ; extra == 'test' - - pyarrow>=13.0.0 ; extra == 'pyarrow' - - bottleneck>=1.4.2 ; extra == 'performance' - - numba>=0.60.0 ; extra == 'performance' - - numexpr>=2.10.2 ; extra == 'performance' - - scipy>=1.14.1 ; extra == 'computation' - - xarray>=2024.10.0 ; extra == 'computation' - - fsspec>=2024.10.0 ; extra == 'fss' - - s3fs>=2024.10.0 ; extra == 'aws' - - gcsfs>=2024.10.0 ; extra == 'gcp' - - odfpy>=1.4.1 ; extra == 'excel' - - openpyxl>=3.1.5 ; extra == 'excel' - - python-calamine>=0.3.0 ; extra == 'excel' - - pyxlsb>=1.0.10 ; extra == 'excel' - - xlrd>=2.0.1 ; extra == 'excel' - - xlsxwriter>=3.2.0 ; extra == 'excel' - - pyarrow>=13.0.0 ; extra == 'parquet' - - pyarrow>=13.0.0 ; extra == 'feather' - - pyiceberg>=0.8.1 ; extra == 'iceberg' - - tables>=3.10.1 ; extra == 'hdf5' - - pyreadstat>=1.2.8 ; extra == 'spss' - - sqlalchemy>=2.0.36 ; extra == 'postgresql' - - psycopg2>=2.9.10 ; extra == 'postgresql' - - adbc-driver-postgresql>=1.2.0 ; extra == 'postgresql' - - sqlalchemy>=2.0.36 ; extra == 'mysql' - - pymysql>=1.1.1 ; extra == 'mysql' - - sqlalchemy>=2.0.36 ; extra == 'sql-other' - - adbc-driver-postgresql>=1.2.0 ; extra == 'sql-other' - - adbc-driver-sqlite>=1.2.0 ; extra == 'sql-other' - - beautifulsoup4>=4.12.3 ; extra == 'html' - - html5lib>=1.1 ; extra == 'html' - - lxml>=5.3.0 ; extra == 'html' - - lxml>=5.3.0 ; extra == 'xml' - - matplotlib>=3.9.3 ; extra == 'plot' - - jinja2>=3.1.5 ; extra == 'output-formatting' - - tabulate>=0.9.0 ; extra == 'output-formatting' - - pyqt5>=5.15.9 ; extra == 'clipboard' - - qtpy>=2.4.2 ; extra == 'clipboard' - - zstandard>=0.23.0 ; extra == 'compression' - - pytz>=2020.1 ; extra == 'timezone' - - adbc-driver-postgresql>=1.2.0 ; extra == 'all' - - adbc-driver-sqlite>=1.2.0 ; extra == 'all' - - beautifulsoup4>=4.12.3 ; extra == 'all' - - bottleneck>=1.4.2 ; extra == 'all' - - fastparquet>=2024.11.0 ; extra == 'all' - - fsspec>=2024.10.0 ; extra == 'all' - - gcsfs>=2024.10.0 ; extra == 'all' - - html5lib>=1.1 ; extra == 'all' - - hypothesis>=6.116.0 ; extra == 'all' - - jinja2>=3.1.5 ; extra == 'all' - - lxml>=5.3.0 ; extra == 'all' - - matplotlib>=3.9.3 ; extra == 'all' - - numba>=0.60.0 ; extra == 'all' - - numexpr>=2.10.2 ; extra == 'all' - - odfpy>=1.4.1 ; extra == 'all' - - openpyxl>=3.1.5 ; extra == 'all' - - psycopg2>=2.9.10 ; extra == 'all' - - pyarrow>=13.0.0 ; extra == 'all' - - pyiceberg>=0.8.1 ; extra == 'all' - - pymysql>=1.1.1 ; extra == 'all' - - pyqt5>=5.15.9 ; extra == 'all' - - pyreadstat>=1.2.8 ; extra == 'all' - - pytest>=8.3.4 ; extra == 'all' - - pytest-xdist>=3.6.1 ; extra == 'all' - - python-calamine>=0.3.0 ; extra == 'all' - - pytz>=2020.1 ; extra == 'all' - - pyxlsb>=1.0.10 ; extra == 'all' - - qtpy>=2.4.2 ; extra == 'all' - - scipy>=1.14.1 ; extra == 'all' - - s3fs>=2024.10.0 ; extra == 'all' - - sqlalchemy>=2.0.36 ; extra == 'all' - - tables>=3.10.1 ; extra == 'all' - - tabulate>=0.9.0 ; extra == 'all' - - xarray>=2024.10.0 ; extra == 'all' - - xlrd>=2.0.1 ; extra == 'all' - - xlsxwriter>=3.2.0 ; extra == 'all' - - zstandard>=0.23.0 ; extra == 'all' - requires_python: '>=3.11' - pypi: https://files.pythonhosted.org/packages/6a/19/ca10ead60b0acc80b2b833c2c4a4f2ff753d0f58b811f70d911c7e94a25c/scipy-1.18.0-cp312-cp312-macosx_10_15_x86_64.whl name: scipy version: 1.18.0 @@ -4662,70 +4284,6 @@ packages: version: 1.6.0 sha256: 1a99710fbb225d459d66def4dc2bb2cd4a9a0bdc8b799fc0621cfdd863be9c93 requires_python: '>=3.10,<3.14' -- pypi: https://files.pythonhosted.org/packages/6c/c2/63fdda36c56437eeb44aaf9493c8bcd62ce230ab1598924fc626ffbfa943/scikit_learn-1.9.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - name: scikit-learn - version: 1.9.0 - sha256: 056c92bb67ad4c28463c2f2653d9701449201e7e7a9e94e321be0f71c4fef2b8 - requires_dist: - - numpy>=1.24.1 - - scipy>=1.10.0 - - joblib>=1.4.0 - - narwhals>=2.0.1 - - threadpoolctl>=3.5.0 - - numpy>=1.24.1 ; extra == 'build' - - scipy>=1.10.0 ; extra == 'build' - - cython>=3.1.2 ; extra == 'build' - - meson-python>=0.17.1 ; extra == 'build' - - numpy>=1.24.1 ; extra == 'install' - - scipy>=1.10.0 ; extra == 'install' - - joblib>=1.4.0 ; extra == 'install' - - narwhals>=2.0.1 ; extra == 'install' - - threadpoolctl>=3.5.0 ; extra == 'install' - - matplotlib>=3.6.1 ; extra == 'benchmark' - - pandas>=1.5.0 ; extra == 'benchmark' - - memory-profiler>=0.57.0 ; extra == 'benchmark' - - matplotlib>=3.6.1 ; extra == 'docs' - - scikit-image>=0.22.0 ; extra == 'docs' - - pandas>=1.5.0 ; extra == 'docs' - - rich>=14.1.0 ; extra == 'docs' - - seaborn>=0.13.0 ; extra == 'docs' - - memory-profiler>=0.57.0 ; extra == 'docs' - - sphinx>=7.3.7 ; extra == 'docs' - - sphinx-copybutton>=0.5.2 ; extra == 'docs' - - sphinx-gallery>=0.17.1 ; extra == 'docs' - - numpydoc>=1.2.0 ; extra == 'docs' - - pillow>=12.1.1 ; extra == 'docs' - - pooch>=1.8.0 ; extra == 'docs' - - sphinx-prompt>=1.4.0 ; extra == 'docs' - - sphinxext-opengraph>=0.9.1 ; extra == 'docs' - - plotly>=5.22.0 ; extra == 'docs' - - polars>=0.20.30 ; extra == 'docs' - - sphinx-design>=0.6.0 ; extra == 'docs' - - sphinxcontrib-sass>=0.3.4 ; extra == 'docs' - - pydata-sphinx-theme>=0.15.3 ; extra == 'docs' - - sphinx-remove-toctrees>=1.0.0.post1 ; extra == 'docs' - - towncrier>=24.8.0 ; extra == 'docs' - - matplotlib>=3.6.1 ; extra == 'examples' - - scikit-image>=0.22.0 ; extra == 'examples' - - pandas>=1.5.0 ; extra == 'examples' - - rich>=14.1.0 ; extra == 'examples' - - seaborn>=0.13.0 ; extra == 'examples' - - pooch>=1.8.0 ; extra == 'examples' - - plotly>=5.22.0 ; extra == 'examples' - - matplotlib>=3.6.1 ; extra == 'tests' - - pandas>=1.5.0 ; extra == 'tests' - - rich>=14.1.0 ; extra == 'tests' - - pytest>=7.1.2 ; extra == 'tests' - - pytest-cov>=2.9.0 ; extra == 'tests' - - ruff>=0.12.2 ; extra == 'tests' - - mypy>=1.15 ; extra == 'tests' - - pyamg>=5.0.0 ; extra == 'tests' - - polars>=0.20.30 ; extra == 'tests' - - pyarrow>=13.0.0 ; extra == 'tests' - - numpydoc>=1.2.0 ; extra == 'tests' - - pooch>=1.8.0 ; extra == 'tests' - - conda-lock==3.0.1 ; extra == 'maintenance' - requires_python: '>=3.11' - pypi: https://files.pythonhosted.org/packages/72/65/a4559d4ff4a210d2a26d7ca0712e4e738cbfc963135f86e8ae807ca420eb/pysam-0.24.0-cp314-cp314-macosx_11_0_arm64.whl name: pysam version: 0.24.0 @@ -4814,11 +4372,6 @@ packages: version: 0.3.25 sha256: 852c41f00afc610322eb3c5aba74068d05b340d09db4573db2deca330ad48f30 requires_python: '>=3.9' -- pypi: https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl - name: joblib - version: 1.5.3 - sha256: 5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713 - requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/7c/03/b8cdb625a21f710dfa11bbca1f48fb4057d2c0286975f8b415bf80942c99/matplotlib-3.11.0-cp314-cp314-macosx_11_0_arm64.whl name: matplotlib version: 3.11.0 @@ -5330,70 +4883,6 @@ packages: name: deeptoolsintervals version: 0.1.9 sha256: 7d94c36fd2b6f10d8b99e536d2672e8228971f1fc810497d33527bba2c40d4f6 -- pypi: https://files.pythonhosted.org/packages/ac/20/75f915ff375d6249e6550ac740fdbbd66159a068fd3af1400ff62036b07a/scikit_learn-1.9.0-cp312-cp312-macosx_10_13_x86_64.whl - name: scikit-learn - version: 1.9.0 - sha256: 2bd41b0d201bc81575531b96b713d3eb5e5f50fb0b82101ff0f92294fdc236ac - requires_dist: - - numpy>=1.24.1 - - scipy>=1.10.0 - - joblib>=1.4.0 - - narwhals>=2.0.1 - - threadpoolctl>=3.5.0 - - numpy>=1.24.1 ; extra == 'build' - - scipy>=1.10.0 ; extra == 'build' - - cython>=3.1.2 ; extra == 'build' - - meson-python>=0.17.1 ; extra == 'build' - - numpy>=1.24.1 ; extra == 'install' - - scipy>=1.10.0 ; extra == 'install' - - joblib>=1.4.0 ; extra == 'install' - - narwhals>=2.0.1 ; extra == 'install' - - threadpoolctl>=3.5.0 ; extra == 'install' - - matplotlib>=3.6.1 ; extra == 'benchmark' - - pandas>=1.5.0 ; extra == 'benchmark' - - memory-profiler>=0.57.0 ; extra == 'benchmark' - - matplotlib>=3.6.1 ; extra == 'docs' - - scikit-image>=0.22.0 ; extra == 'docs' - - pandas>=1.5.0 ; extra == 'docs' - - rich>=14.1.0 ; extra == 'docs' - - seaborn>=0.13.0 ; extra == 'docs' - - memory-profiler>=0.57.0 ; extra == 'docs' - - sphinx>=7.3.7 ; extra == 'docs' - - sphinx-copybutton>=0.5.2 ; extra == 'docs' - - sphinx-gallery>=0.17.1 ; extra == 'docs' - - numpydoc>=1.2.0 ; extra == 'docs' - - pillow>=12.1.1 ; extra == 'docs' - - pooch>=1.8.0 ; extra == 'docs' - - sphinx-prompt>=1.4.0 ; extra == 'docs' - - sphinxext-opengraph>=0.9.1 ; extra == 'docs' - - plotly>=5.22.0 ; extra == 'docs' - - polars>=0.20.30 ; extra == 'docs' - - sphinx-design>=0.6.0 ; extra == 'docs' - - sphinxcontrib-sass>=0.3.4 ; extra == 'docs' - - pydata-sphinx-theme>=0.15.3 ; extra == 'docs' - - sphinx-remove-toctrees>=1.0.0.post1 ; extra == 'docs' - - towncrier>=24.8.0 ; extra == 'docs' - - matplotlib>=3.6.1 ; extra == 'examples' - - scikit-image>=0.22.0 ; extra == 'examples' - - pandas>=1.5.0 ; extra == 'examples' - - rich>=14.1.0 ; extra == 'examples' - - seaborn>=0.13.0 ; extra == 'examples' - - pooch>=1.8.0 ; extra == 'examples' - - plotly>=5.22.0 ; extra == 'examples' - - matplotlib>=3.6.1 ; extra == 'tests' - - pandas>=1.5.0 ; extra == 'tests' - - rich>=14.1.0 ; extra == 'tests' - - pytest>=7.1.2 ; extra == 'tests' - - pytest-cov>=2.9.0 ; extra == 'tests' - - ruff>=0.12.2 ; extra == 'tests' - - mypy>=1.15 ; extra == 'tests' - - pyamg>=5.0.0 ; extra == 'tests' - - polars>=0.20.30 ; extra == 'tests' - - pyarrow>=13.0.0 ; extra == 'tests' - - numpydoc>=1.2.0 ; extra == 'tests' - - pooch>=1.8.0 ; extra == 'tests' - - conda-lock==3.0.1 ; extra == 'maintenance' - requires_python: '>=3.11' - pypi: https://files.pythonhosted.org/packages/ac/3e/61d11b779e4e83ee8661439af070f6f138060cd41f0346a0a70432f9e55c/cwl_upgrader-1.2.15-py3-none-any.whl name: cwl-upgrader version: 1.2.15 diff --git a/pydeeptools/deeptools/alignmentSieve2.py b/pydeeptools/deeptools/alignmentSieve2.py index 55fe0b0fbc..d230f7cd22 100644 --- a/pydeeptools/deeptools/alignmentSieve2.py +++ b/pydeeptools/deeptools/alignmentSieve2.py @@ -43,9 +43,21 @@ def parseArguments(): help="The number of entries in total and filtered are saved to this file") general.add_argument('--filteredOutReads', - metavar="filtered.bam", - default="None", - help="If desired, all reads NOT passing the filtering criteria can be written to this file.") + metavar="filtered.bam", + default="None", + help="If desired, all reads NOT passing the filtering criteria can be written to this file.") + + general.add_argument('--label', '-l', + metavar='sample1', + default="None", + help='User defined label instead of the default label ' + '(file name).') + + general.add_argument('--smartLabels', + action='store_true', + help='Instead of manually specifying a labels for the input ' + 'file, this causes deepTools to use the file name ' + 'after removing the path and extension.') general.add_argument('--verbose', '-v', help='Set to see processing messages.', @@ -112,6 +124,12 @@ def parseArguments(): default="None", required=False) + filtering.add_argument('--ignoreDuplicates', + help='If set, reads that are marked as PCR ' + 'or optical duplicates (SAM flag 0x400) will ' + 'be filtered out.', + action='store_true') + filtering.add_argument('--minFragmentLength', help='The minimum fragment length needed for read/pair ' 'inclusion. This option is primarily useful ' @@ -340,8 +358,15 @@ def main(args=None): print("Warning! The --ATACshift option is used, but a --shift option is provided as well. The latter will be ignored in favor of 4 -5 5 -4.") args.shift = [4, -5, 5, -4] - # Remove args: - # label, smartLabels, genomeChunkLength, ignoreDuplicates. + if args.ignoreDuplicates: + args.samFlagExclude |= 0x400 + + if not args.blackListFileName: + args.blackListFileName = "None" + elif isinstance(args.blackListFileName, list): + if len(args.blackListFileName) != 1: + sys.exit("Only one blacklist file is supported when using '--alignmentsieve rust'.") + args.blackListFileName = args.blackListFileName[0] signal.signal(signal.SIGINT, signal.SIG_DFL) r_alignmentsieve( @@ -360,4 +385,6 @@ def main(args=None): args.blackListFileName, args.minFragmentLength, args.maxFragmentLength, + args.label, + args.smartLabels, ) diff --git a/pydeeptools/deeptools/bamCoverage2.py b/pydeeptools/deeptools/bamCoverage2.py index deac6d428a..3e44c9aa8b 100644 --- a/pydeeptools/deeptools/bamCoverage2.py +++ b/pydeeptools/deeptools/bamCoverage2.py @@ -211,5 +211,5 @@ def main(args=None): args.numberOfProcessors, # threads args.region, # regions args.verbose, # verbose - args.no_collapse, - ) \ No newline at end of file + args.no_collapse, + ) diff --git a/pydeeptools/deeptools/bamPEFragmentSize.py b/pydeeptools/deeptools/bamPEFragmentSize.py index 362b118555..3f6db136c1 100755 --- a/pydeeptools/deeptools/bamPEFragmentSize.py +++ b/pydeeptools/deeptools/bamPEFragmentSize.py @@ -3,17 +3,11 @@ import argparse import sys - import matplotlib import numpy as np - -matplotlib.use("Agg") -matplotlib.rcParams["pdf.fonttype"] = 42 -matplotlib.rcParams["svg.fonttype"] = "none" +from deeptools import matplotlib_defaults from importlib.metadata import version - import matplotlib.pyplot as plt - from deeptools.getFragmentAndReadSize import get_read_and_fragment_length # own tools diff --git a/pydeeptools/deeptools/correlation.py b/pydeeptools/deeptools/correlation.py index 4d8ad36bc4..77a9ad37a3 100644 --- a/pydeeptools/deeptools/correlation.py +++ b/pydeeptools/deeptools/correlation.py @@ -4,10 +4,7 @@ import numpy as np import scipy.cluster.hierarchy as sch import scipy.stats -import matplotlib as mpl -mpl.use('Agg') -mpl.rcParams['pdf.fonttype'] = 42 -mpl.rcParams['svg.fonttype'] = 'none' +from deeptools import matplotlib_defaults import matplotlib.pyplot as plt import matplotlib.gridspec as gridspec import matplotlib.ticker @@ -246,7 +243,7 @@ def plot_correlation(self, plot_filename, plot_title='', vmax=None, font_size = 5 else: font_size = int(14 - 0.25 * num_rows) - mpl.rcParams.update({'font.size': font_size}) + matplotlib.rcParams.update({'font.size': font_size}) # set the minimum and maximum values if vmax is None: vmax = 1 @@ -348,7 +345,6 @@ def plot_scatter(self, plot_filename, plot_title='', image_format=None, log1p=Fa grids = gridspec.GridSpec(num_samples, num_samples) grids.update(wspace=0, hspace=0) fig = plt.figure(figsize=(2 * num_samples, 2 * num_samples)) - plt.rcParams['font.size'] = 8.0 plt.suptitle(plot_title) if log1p is True: self.matrix = np.log1p(self.matrix) diff --git a/pydeeptools/deeptools/correlation_heatmap.py b/pydeeptools/deeptools/correlation_heatmap.py index 58dbdfbc04..690717ee95 100644 --- a/pydeeptools/deeptools/correlation_heatmap.py +++ b/pydeeptools/deeptools/correlation_heatmap.py @@ -1,15 +1,11 @@ -from matplotlib import use as mplt_use -mplt_use('Agg') +from deeptools import matplotlib_defaults from deeptools import cm # noqa: F401 import matplotlib.pyplot as plt import numpy as np import scipy.cluster.hierarchy as sch -from matplotlib import rcParams import matplotlib.colors as pltcolors import copy -rcParams['pdf.fonttype'] = 42 -rcParams['svg.fonttype'] = 'none' old_settings = np.seterr(all='ignore') @@ -26,7 +22,7 @@ def plot_correlation(corr_matrix, labels, plotFileName, vmax=None, font_size = 5 else: font_size = int(14 - 0.25 * num_rows) - rcParams.update({'font.size': font_size}) + matplotlib.rcParams.update({'font.size': font_size}) # set the minimum and maximum values if vmax is None: vmax = 1 diff --git a/pydeeptools/deeptools/heatmapper_utilities.py b/pydeeptools/deeptools/heatmapper_utilities.py index 39256b5fdc..130de0945f 100644 --- a/pydeeptools/deeptools/heatmapper_utilities.py +++ b/pydeeptools/deeptools/heatmapper_utilities.py @@ -1,8 +1,5 @@ import numpy as np -import matplotlib -matplotlib.use('Agg') -matplotlib.rcParams['pdf.fonttype'] = 42 -matplotlib.rcParams['svg.fonttype'] = 'none' +from deeptools import matplotlib_defaults import matplotlib.colors as pltcolors import textwrap diff --git a/pydeeptools/deeptools/matplotlib_defaults.py b/pydeeptools/deeptools/matplotlib_defaults.py new file mode 100644 index 0000000000..4b62527352 --- /dev/null +++ b/pydeeptools/deeptools/matplotlib_defaults.py @@ -0,0 +1,22 @@ +import matplotlib + +# Backend should be set before importing pyplot +matplotlib.use("Agg") + +# Central matplotlib configuration +matplotlib.rcParams.update({ + + # Output compatibility + "pdf.fonttype": 42, + "ps.fonttype": 42, + "svg.fonttype": "none", + + # Fonts + "font.size": 8.0, + + # Figure defaults + "figure.dpi": 100, + "savefig.dpi": 200, +}) + + diff --git a/pydeeptools/deeptools/plotCorrelation.py b/pydeeptools/deeptools/plotCorrelation.py index c9ec13e670..1aa7118e9a 100644 --- a/pydeeptools/deeptools/plotCorrelation.py +++ b/pydeeptools/deeptools/plotCorrelation.py @@ -4,10 +4,7 @@ import sys import argparse import numpy as np -import matplotlib -matplotlib.use('Agg') -matplotlib.rcParams['pdf.fonttype'] = 42 -matplotlib.rcParams['svg.fonttype'] = 'none' +from deeptools import matplotlib_defaults import matplotlib.pyplot as plt from importlib.metadata import version from deeptools.correlation import Correlation diff --git a/pydeeptools/deeptools/plotCoverage.py b/pydeeptools/deeptools/plotCoverage.py index 0b6df5b4a0..4cf2ca144b 100755 --- a/pydeeptools/deeptools/plotCoverage.py +++ b/pydeeptools/deeptools/plotCoverage.py @@ -5,11 +5,7 @@ import sys import argparse import numpy as np - -import matplotlib -matplotlib.use('Agg') -matplotlib.rcParams['pdf.fonttype'] = 42 -matplotlib.rcParams['svg.fonttype'] = 'none' +from deeptools import matplotlib_defaults import matplotlib.pyplot as plt from importlib.metadata import version import deeptools.countReadsPerBin as countR diff --git a/pydeeptools/deeptools/plotEnrichment.py b/pydeeptools/deeptools/plotEnrichment.py index d1d0c1e75d..26c9d590c8 100755 --- a/pydeeptools/deeptools/plotEnrichment.py +++ b/pydeeptools/deeptools/plotEnrichment.py @@ -4,10 +4,7 @@ import sys import argparse import numpy as np -import matplotlib -matplotlib.use('Agg') -matplotlib.rcParams['pdf.fonttype'] = 42 -matplotlib.rcParams['svg.fonttype'] = 'none' +from deeptools import matplotlib_defaults import matplotlib.pyplot as plt import matplotlib.gridspec as gridspec diff --git a/pydeeptools/deeptools/plotFingerprint.py b/pydeeptools/deeptools/plotFingerprint.py index f0756bc2a8..5600e8c4db 100755 --- a/pydeeptools/deeptools/plotFingerprint.py +++ b/pydeeptools/deeptools/plotFingerprint.py @@ -4,10 +4,7 @@ import sys import argparse import numpy as np -import matplotlib -matplotlib.use('Agg') -matplotlib.rcParams['pdf.fonttype'] = 42 -matplotlib.rcParams['svg.fonttype'] = 'none' +from deeptools import matplotlib_defaults import matplotlib.pyplot as plt from scipy import interpolate from scipy.stats import poisson diff --git a/pydeeptools/deeptools/plotHeatmap.py b/pydeeptools/deeptools/plotHeatmap.py index 5cbbb51550..a23aa8e1ea 100755 --- a/pydeeptools/deeptools/plotHeatmap.py +++ b/pydeeptools/deeptools/plotHeatmap.py @@ -6,9 +6,7 @@ from collections import OrderedDict import numpy as np import matplotlib -matplotlib.use('Agg') -matplotlib.rcParams['pdf.fonttype'] = 42 -matplotlib.rcParams['svg.fonttype'] = 'none' +from deeptools import matplotlib_defaults import matplotlib.pyplot as plt from matplotlib.font_manager import FontProperties import matplotlib.gridspec as gridspec @@ -290,8 +288,9 @@ def plotMatrix(hm, outFileName, yMin = [yMin] if not isinstance(yMax, list): yMax = [yMax] + + plt.rcParams['font.size'] = matplotlib.rcParams['font.size'] - plt.rcParams['font.size'] = 8.0 fontP = FontProperties() showSummaryPlot = False diff --git a/pydeeptools/deeptools/plotPCA.py b/pydeeptools/deeptools/plotPCA.py index 1532706cac..24260b2c79 100644 --- a/pydeeptools/deeptools/plotPCA.py +++ b/pydeeptools/deeptools/plotPCA.py @@ -3,10 +3,7 @@ import sys import argparse -import matplotlib -matplotlib.use('Agg') -matplotlib.rcParams['pdf.fonttype'] = 42 -matplotlib.rcParams['svg.fonttype'] = 'none' +from deeptools import matplotlib_defaults from importlib.metadata import version from deeptools.correlation import Correlation from deeptools.parserCommon import writableFile, expand_list diff --git a/pydeeptools/deeptools/plotProfile.py b/pydeeptools/deeptools/plotProfile.py index 53ed0625b9..0371208c4e 100755 --- a/pydeeptools/deeptools/plotProfile.py +++ b/pydeeptools/deeptools/plotProfile.py @@ -7,10 +7,7 @@ import argparse import numpy as np from math import ceil -import matplotlib -matplotlib.use('Agg') -matplotlib.rcParams['pdf.fonttype'] = 42 -matplotlib.rcParams['svg.fonttype'] = 'none' +from deeptools import matplotlib_defaults import matplotlib.pyplot as plt from matplotlib.font_manager import FontProperties from matplotlib import colors as pltcolors @@ -178,7 +175,6 @@ def __init__(self, hm, out_file_name, cols = self.numplots self.grids = gridspec.GridSpec(rows, cols) - plt.rcParams['font.size'] = 8.0 self.font_p = FontProperties() self.font_p.set_size('small') diff --git a/pydeeptools/deeptools/test/test_alignmentseive.py b/pydeeptools/deeptools/test/test_alignmentseive.py deleted file mode 100644 index b43a19b969..0000000000 --- a/pydeeptools/deeptools/test/test_alignmentseive.py +++ /dev/null @@ -1,122 +0,0 @@ -import deeptools.alignmentSieve as aln_seive - -import os.path -from os import unlink - -ROOT = os.path.dirname(os.path.abspath(__file__)) + "/test_data/" -BAMFILE_IN = ROOT + "paired_chr2L.bam" -CRAMFILE_IN = ROOT + "paired_chr2L.cram" -FILTER_METRICS_FILE = ROOT + "alignmentSieve.txt" -BED_OUT = ROOT + "alignmentSieve.bed" -BAMFILE_OUT = ROOT + "alignmentSieve.bam" -BAMFILE_OUT2 = ROOT + "alignmentSieve2.bam" -BAMFILE_OUT3 = ROOT + "alignmentSieve3.bam" - - -def test_minimum_mapping_quality_filter_metric(): - """ - Test minimal command line args for alignement sieve - """ - outfile = '/tmp/test_bam.bam' - filter_metric_file = '/tmp/test_metrics.txt' - args = "--bam {} -o {} --minMappingQuality 10 --filterMetrics {}".format(BAMFILE_IN, outfile, filter_metric_file).split() - aln_seive.main(args) - - _foo = open(filter_metric_file, "r") - resp = _foo.readlines()[2] - _foo.close() - expected = 'paired_chr2L.bam\t8440\t12644\n' - assert expected in resp, f"'{expected}' not found in '{resp}'" - - bam_file_size = os.path.getsize(BAMFILE_OUT) - expected_file_size = os.path.getsize(outfile) - size_tolerance = 5000 - size_difference = abs(bam_file_size - expected_file_size) - assert size_difference <= size_tolerance, "File size do not match" - unlink(outfile) - unlink(filter_metric_file) - - -def test_fragment_length_filter_metric(): - """ - Test --minFragmentLength / --maxFragmentLength filtering (fragment-size - selection, e.g. nucleosome-free vs mono-nucleosome). Only read pairs whose - template length falls in [130, 200] are kept. - """ - outfile = '/tmp/test_bam.bam' - filter_metric_file = '/tmp/test_metrics.txt' - args = "--bam {} -o {} --minFragmentLength 130 --maxFragmentLength 200 --filterMetrics {}".format(BAMFILE_IN, outfile, filter_metric_file).split() - aln_seive.main(args) - - _foo = open(filter_metric_file, "r") - resp = _foo.readlines()[2] - _foo.close() - expected = 'paired_chr2L.bam\t7933\t12644\n' - assert expected in resp, f"'{expected}' not found in '{resp}'" - unlink(outfile) - unlink(filter_metric_file) - - -def test_filterRNAstrand_forward_filter_metric(): - """ - Test --filterRNAstrand forward (keep only reads matching the forward - transcription strand, using paired-end mate orientation). - """ - outfile = '/tmp/test_bam.bam' - filter_metric_file = '/tmp/test_metrics.txt' - args = "--bam {} -o {} --filterRNAstrand forward --filterMetrics {}".format(BAMFILE_IN, outfile, filter_metric_file).split() - aln_seive.main(args) - - _foo = open(filter_metric_file, "r") - resp = _foo.readlines()[2] - _foo.close() - expected = 'paired_chr2L.bam\t6303\t12644\n' - assert expected in resp, f"'{expected}' not found in '{resp}'" - unlink(outfile) - unlink(filter_metric_file) - - -def test_with_bed_output_along_with_shifts(): - """ - Tests Alignment seive with shifts and output BED file - """ - output_bed_file = "/tmp/aln_seive.bed" - args = "--bam {} -o {} --minMappingQuality 10 --BED --shift 1 -2 3 -4".format(BAMFILE_IN, output_bed_file).split() - aln_seive.main(args) - with open(output_bed_file, "r") as _foo: - result = len(_foo.readlines()) - _expected = 4261 - assert result == _expected, "No of lines in BED files differ" - unlink(output_bed_file) - - -def test_with_bam_output_with_shifts(): - """ - Tests Alignment seive with shifts and output BAM file - """ - output_bam_file = "/tmp/aln_seive.bam" - args = "--bam {} -o {} --minMappingQuality 10 --shift 1 -2 3 -4".format(BAMFILE_IN, output_bam_file).split() - aln_seive.main(args) - - bam_file_size = os.path.getsize(BAMFILE_OUT2) - expected_file_size = os.path.getsize(output_bam_file) - size_tolerance = 5000 - size_difference = abs(bam_file_size - expected_file_size) - assert size_difference <= size_tolerance, "File sizes do not match" - unlink(output_bam_file) - - -def test_with_cram_output_with_shifts(): - """ - Tests Alignment seive with CRAM input along with shifts - """ - output_bam_file = "/tmp/aln_seive2.bam" - args = "--bam {} -o {} --minMappingQuality 10 --shift 1 -2 3 -4".format(BAMFILE_IN, output_bam_file).split() - aln_seive.main(args) - - bam_file_size = os.path.getsize(BAMFILE_OUT3) - expected_file_size = os.path.getsize(output_bam_file) - size_tolerance = 5000 - size_difference = abs(bam_file_size - expected_file_size) - assert size_difference <= size_tolerance, "File sizes do not match" - unlink(output_bam_file) diff --git a/pydeeptools/deeptools/test/test_alignmentsieve.py b/pydeeptools/deeptools/test/test_alignmentsieve.py new file mode 100644 index 0000000000..cd98e5a827 --- /dev/null +++ b/pydeeptools/deeptools/test/test_alignmentsieve.py @@ -0,0 +1,147 @@ +import deeptools.alignmentSieve2 as aln_seive +import tempfile +from pathlib import Path +import pysam + +ROOT = Path(__file__).parent / "test_data" +BAMFILE_IN = str(ROOT / "paired_chr2L.bam") +CRAMFILE_IN = str(ROOT / "paired_chr2L.cram") +BL_IN = str(ROOT / "alignmentSieve_blacklist.bed") +FILTER_METRICS_FILE = str(ROOT / "alignmentSieve.txt") +BED_OUT = str(ROOT / "alignmentSieve.bed") +BEDFILE = str(ROOT / "alignmentSieve.bed") +BAMFILE_OUT1 = str(ROOT / "alignmentSieve1.bam") +BAMFILEFILT_OUT1 = str(ROOT / "alignmentSieve1.filtered.bam") +BAMFILE_OUT2 = str(ROOT / "alignmentSieve2.bam") +BAMFILE_OUT3 = str(ROOT / "alignmentSieve3.bam") +BAMFILE_OUT4 = str(ROOT / "alignmentSieve4.bam") +BAMFILE_OUT5 = str(ROOT / "alignmentSieve5.bam") +BAMFILE_OUT6 = str(ROOT / "alignmentSieve6.bam") +BAMFILE_OUT7 = str(ROOT / "alignmentSieve7.bam") + +def _assert_equals(expected: str, actual: str) -> None: + if Path(expected).suffix == '.bam': + with ( + pysam.AlignmentFile(expected, "rb") as ef, + pysam.AlignmentFile(actual, "rb") as af, + ): + for eread, aread in zip(ef, af): + assert eread.to_string() == aread.to_string(), f"BAM files do not match: {eread} vs {aread}" + else: + with ( + open(expected, 'r') as ef, + open(actual, 'r') as af, + ): + for eread, aread in zip(ef, af): + assert eread == aread, f"files do not match: {eread} vs {aread}" + +def test_alsieve_minmapq(): + """ + Test minimal command line args for alignement sieve + """ + _, outfile = tempfile.mkstemp(suffix=".bam") + _, filter_metric_file = tempfile.mkstemp(suffix=".txt") + _, filtered_bamfile = tempfile.mkstemp(suffix=".bam") + args = f"--bam {BAMFILE_IN} -o {outfile} --minMappingQuality 10 --filteredOutReads {filtered_bamfile} --filterMetrics {filter_metric_file}".split() + aln_seive.main(args) + + _foo = open(filter_metric_file, "r") + resp = _foo.readlines()[2] + _foo.close() + expected = 'paired_chr2L.bam\t8440\t12644\n' + assert expected in resp, f"'{expected}' not found in '{resp}'" + + _assert_equals(BAMFILE_OUT1, outfile) + + +def test_alsieve_fraglen(): + """ + Test --minFragmentLength / --maxFragmentLength filtering (fragment-size + selection, e.g. nucleosome-free vs mono-nucleosome). Only read pairs whose + template length falls in [130, 200] are kept. + """ + _, outfile = tempfile.mkstemp(suffix=".bam") + _, filter_metric_file = tempfile.mkstemp(suffix=".txt") + args = f"--bam {BAMFILE_IN} -o {outfile} --minFragmentLength 130 --maxFragmentLength 200 --filterMetrics {filter_metric_file}".split() + aln_seive.main(args) + + _foo = open(filter_metric_file, "r") + resp = _foo.readlines()[2] + _foo.close() + expected = 'paired_chr2L.bam\t7933\t12644\n' + assert expected in resp, f"'{expected}' not found in '{resp}'" + _assert_equals(BAMFILE_OUT2, outfile) + + +def test_alsieve_rnastrand(): + """ + Test --filterRNAstrand forward (keep only reads matching the forward + transcription strand, using paired-end mate orientation). + """ + _, outfile = tempfile.mkstemp(suffix=".bam") + _, filter_metric_file = tempfile.mkstemp(suffix=".txt") + args = f"--bam {BAMFILE_IN} -o {outfile} --filterRNAstrand forward --filterMetrics {filter_metric_file}".split() + aln_seive.main(args) + + _foo = open(filter_metric_file, "r") + resp = _foo.readlines()[2] + _foo.close() + expected = 'paired_chr2L.bam\t6303\t12644\n' + assert expected in resp, f"'{expected}' not found in '{resp}'" + _assert_equals(BAMFILE_OUT3, outfile) + + +def test_alsieve_shift_bed(): + """ + Tests Alignment seive with shifts and output BED file + """ + _, output_bed_file = tempfile.mkstemp(suffix=".bed") + args = f"--bam {BAMFILE_IN} -o {output_bed_file} --minMappingQuality 10 --BED --shift 1 -2 3 -4".split() + aln_seive.main(args) + with open(output_bed_file, "r") as _foo: + result = len(_foo.readlines()) + _expected = 4261 + assert result == _expected, "No of lines in BED files differ" + _assert_equals(BEDFILE, output_bed_file) + + +def test_alsieve_shift(): + """ + Tests Alignment seive with shifts and output BAM file + """ + _, output_bam_file = tempfile.mkstemp(suffix=".bam") + args = f"--bam {BAMFILE_IN} -o {output_bam_file} --minMappingQuality 10 --shift 1 -2 3 -4".split() + aln_seive.main(args) + + _assert_equals(BAMFILE_OUT4, output_bam_file) + + +def test_alsieve_blacklist(): + """ + Tests Alignment seive with shifts and output BAM file + """ + _, output_bam_file = tempfile.mkstemp(suffix=".bam") + args = f"--bam {BAMFILE_IN} -bl {BL_IN} -o {output_bam_file} --minMappingQuality 10 --shift 1 -2 3 -4".split() + aln_seive.main(args) + + _assert_equals(BAMFILE_OUT5, output_bam_file) + +def test_alsieve_shift2(): + """ + shift with 2 arguments. + """ + _, output_bam_file = tempfile.mkstemp(suffix=".bam") + args = f"--bam {BAMFILE_IN} -o {output_bam_file} --shift 1 -2".split() + aln_seive.main(args) + + _assert_equals(BAMFILE_OUT6, output_bam_file) + +def test_alsieve_atacshift(): + """ + Tests Alignment seive with ATAC shift + """ + _, output_bam_file = tempfile.mkstemp(suffix=".bam") + args = f"--bam {BAMFILE_IN} -o {output_bam_file} --minMappingQuality 10 --ATACshift".split() + aln_seive.main(args) + + _assert_equals(BAMFILE_OUT7, output_bam_file) diff --git a/pydeeptools/deeptools/test/test_bamCoverage_and_bamCompare.py b/pydeeptools/deeptools/test/test_bamCoverage_and_bamCompare.py index 492d72426c..06098b9317 100644 --- a/pydeeptools/deeptools/test/test_bamCoverage_and_bamCompare.py +++ b/pydeeptools/deeptools/test/test_bamCoverage_and_bamCompare.py @@ -3,6 +3,7 @@ import os.path import filecmp from os import unlink +import tempfile ROOT = os.path.dirname(os.path.abspath(__file__)) + "/test_data/" BAMFILE_A = ROOT + "testA.bam" @@ -35,7 +36,7 @@ def test_bam_coverage_arguments(): """ Test minimal command line args for bamCoverage """ - outfile = '/tmp/test_file.bg' + _, outfile = tempfile.mkstemp(suffix=".bg") #for fname in [BAMFILE_B, CRAMFILE_B]: for fname in [BAMFILE_B]: args = "--bam {} -o {} --outFileFormat bedgraph".format(fname, outfile).split() @@ -50,7 +51,7 @@ def test_bam_coverage_arguments(): def test_bam_coverage_extend(): - outfile = '/tmp/test_file.bg' + _, outfile = tempfile.mkstemp(suffix=".bg") #for fname in [BAMFILE_B, CRAMFILE_B]: for fname in [BAMFILE_B]: args = "-b {} -o {} --extendReads 100 --outFileFormat bedgraph".format(fname, outfile).split() @@ -65,7 +66,7 @@ def test_bam_coverage_extend(): def test_bam_coverage_extend_and_normalizeUsingRPGC(): - outfile = '/tmp/test_file.bg' + _, outfile = tempfile.mkstemp(suffix=".bg") #for fname in [BAMFILE_B, CRAMFILE_B]: for fname in [BAMFILE_B]: args = "-b {} -o {} --normalizeUsing RPGC --effectiveGenomeSize 200 --extendReads 100 --verbose " \ @@ -82,7 +83,7 @@ def test_bam_coverage_extend_and_normalizeUsingRPGC(): def test_bam_coverage_skipnas(): - outfile = '/tmp/test_file.bg' + _, outfile = tempfile.mkstemp(suffix=".bg") #for fname in [BAMFILE_B, CRAMFILE_B]: for fname in [BAMFILE_B]: args = "--bam {} -o {} --outFileFormat bedgraph --skipNAs".format(fname, outfile).split() @@ -102,7 +103,7 @@ def test_bam_coverage_normalizeUsingRPKM(): read), so each covered read contributes 1e9 / (binLength_kb * totalReads) reads-per-kilobase-per-million. """ - outfile = '/tmp/test_file.bg' + _, outfile = tempfile.mkstemp(suffix=".bg") for fname in [BAMFILE_B]: args = "--bam {} -o {} --outFileFormat bedgraph --normalizeUsing RPKM".format(fname, outfile).split() bam_cov.main(args) @@ -120,7 +121,7 @@ def test_bam_coverage_scaleFactor(): Test --scaleFactor. A manual scale factor is applied directly to the raw coverage (and takes precedence over --normalizeUsing). """ - outfile = '/tmp/test_file.bg' + _, outfile = tempfile.mkstemp(suffix=".bg") for fname in [BAMFILE_B]: args = "--bam {} -o {} --outFileFormat bedgraph --scaleFactor 2.0".format(fname, outfile).split() bam_cov.main(args) @@ -134,7 +135,7 @@ def test_bam_coverage_scaleFactor(): # def test_bam_coverage_filtering(): -# outfile = '/tmp/test_file.bg' +# _, outfile = tempfile.mkstemp(suffix=".bg") # #for fname in [BAMFILE_B, CRAMFILE_B]: # for fname in [BAMFILE_B]: # args = "--bam {} -o {} --outFileFormat bedgraph --ignoreDuplicates --verbose".format(fname, outfile).split() @@ -154,7 +155,7 @@ def test_bam_compare_arguments(): between the same file is taken, therefore, the expected value is 1.0 for all bins. """ - outfile = '/tmp/test_file.bg' + _, outfile = tempfile.mkstemp(suffix=".bg") #for fname in [BAMFILE_B, CRAMFILE_B]: for fname in [BAMFILE_B]: args = "--bamfile1 {} --bamfile2 {} " \ @@ -173,7 +174,7 @@ def test_bam_compare_diff_files(): """ Test with two different files """ - outfile = '/tmp/test_file.bg' + _, outfile = tempfile.mkstemp(suffix=".bg") #for A, B in [(BAMFILE_A, BAMFILE_B), (CRAMFILE_A, CRAMFILE_B)]: for A, B in [(BAMFILE_A, BAMFILE_B)]: args = "--bamfile1 {} --bamfile2 {} --scaleFactors 1:1 --operation subtract --verbose " \ @@ -192,7 +193,7 @@ def test_bam_compare_pseudocounts(): """ Test with different pseudocounts """ - outfile = '/tmp/test_file.bg' + _, outfile = tempfile.mkstemp(suffix=".bg") args = "--bamfile1 {} --bamfile2 {} --outFileFormat bedgraph --scaleFactors 1:1 -o {} " \ "--pseudocount 1 0".format(BAMFILE_A, BAMFILE_B, outfile).split() bam_comp.main(args) @@ -209,7 +210,7 @@ def test_bam_compare_ZoverZ(): """ Ensure --skipZeroOverZero works in bamCompare """ - outfile = '/tmp/test_file.bg' + _, outfile = tempfile.mkstemp(suffix=".bg") args = "--bamfile1 {} --bamfile2 {} --outFileFormat bedgraph --scaleFactors 1:1 -o {} " \ "--skipZeroOverZero --verbose".format(BAMFILE_A, BAMFILE_B, outfile).split() bam_comp.main(args) @@ -260,7 +261,7 @@ def test_bam_compare_diff_files_skipnas(): Compared to the previous tests, any region that do not have coverage (in either of the bam files) is not included in the bedgraph file. """ - outfile = '/tmp/test_file.bg' + _, outfile = tempfile.mkstemp(suffix=".bg") #for A, B in [(BAMFILE_A, BAMFILE_B), (CRAMFILE_A, CRAMFILE_B)]: for A, B in [(BAMFILE_A, BAMFILE_B)]: args = "--bamfile1 {} --bamfile2 {} --scaleFactors 1:1 --operation subtract " \ @@ -279,7 +280,7 @@ def test_bam_compare_extend(): """ Test read extension """ - outfile = '/tmp/test_file.bg' + _, outfile = tempfile.mkstemp(suffix=".bg") #for A, B in [(BAMFILE_A, BAMFILE_B), (CRAMFILE_A, CRAMFILE_B)]: for A, B in [(BAMFILE_A, BAMFILE_B)]: args = "--bamfile1 {} --bamfile2 {} --extend 100 --scaleFactors 1:1 --operation subtract " \ @@ -298,7 +299,7 @@ def test_bam_compare_scale_factors_ratio(): """ Test scale factor """ - outfile = '/tmp/test_file.bg' + _, outfile = tempfile.mkstemp(suffix=".bg") #for A, B in [(BAMFILE_A, BAMFILE_B), (CRAMFILE_A, CRAMFILE_B)]: for A, B in [(BAMFILE_A, BAMFILE_B)]: args = "--bamfile1 {} --bamfile2 {} --operation ratio --ignoreForNormalization chr_cigar " \ @@ -339,7 +340,7 @@ def test_bam_compare_scale_factors_subtract(): """ Test scale factor """ - outfile = '/tmp/test_file.bg' + _, outfile = tempfile.mkstemp(suffix=".bg") #for A, B in [(BAMFILE_A, BAMFILE_B), (CRAMFILE_A, CRAMFILE_B)]: for A, B in [(BAMFILE_A, BAMFILE_B)]: args = "--bamfile1 {} --bamfile2 {} --operation subtract --ignoreForNormalization chr_cigar " \ @@ -382,7 +383,7 @@ def test_bam_coverage_filter_blacklist(): """ Test --samFlagInclude --samFlagExclude --minMappingQuality and --blackListFileName """ - outfile = '/tmp/test_file_filter.bg' + _, outfile = tempfile.mkstemp(suffix=".bg") #for fname in [BAMFILE_FILTER1, CRAMFILE_FILTER1]: for fname in [BAMFILE_FILTER1]: args = "--bam {} --normalizeUsing RPGC --effectiveGenomeSize 1400 -p 1 -o {} -of bedgraph --samFlagInclude 512 " \ @@ -424,7 +425,7 @@ def test_bam_coverage_offset1(): """ Test -bs 1 --Offset 1 """ - outfile = '/tmp/test_offset.bw' + _, outfile = tempfile.mkstemp(suffix=".bw") #for fname in [BAMFILE_A, CRAMFILE_A]: for fname in [BAMFILE_A]: args = "--Offset 1 --bam {} -p 1 -bs 1 -o {} -of bedgraph --verbose ".format(fname, outfile) @@ -453,7 +454,7 @@ def test_bam_coverage_offset1_10(): """ Test -bs 1 --Offset 1 10 """ - outfile = '/tmp/test_offset.bw' + _, outfile = tempfile.mkstemp(suffix=".bw") #for fname in [BAMFILE_A, CRAMFILE_A]: for fname in [BAMFILE_A]: args = "--Offset 1 10 -b {} -p 1 -bs 1 -of bedgraph -o {}".format(fname, outfile) @@ -481,7 +482,7 @@ def test_bam_coverage_offset_minus1(): """ Test -bs 1 --Offset -1 """ - outfile = '/tmp/test_offset.bw' + _, outfile = tempfile.mkstemp(suffix=".bw") #for fname in [BAMFILE_A, CRAMFILE_A]: for fname in [BAMFILE_A]: args = "--Offset -1 -b {} -p 1 -bs 1 -of bedgraph -o {}".format(fname, outfile) @@ -508,7 +509,7 @@ def test_bam_coverage_offset20_minus4(): """ Test -bs 1 --Offset 20 -4 """ - outfile = '/tmp/test_offset.bw' + _, outfile = tempfile.mkstemp(suffix=".bw") #for fname in [BAMFILE_A, CRAMFILE_A]: for fname in [BAMFILE_A]: args = "--Offset 20 -4 -b {} -p 1 -bs 1 -of bedgraph -o {}".format(fname, outfile) @@ -539,7 +540,7 @@ def test_bam_compare_filter_blacklist(): """ Test --samFlagInclude --samFlagExclude --minMappingQuality --ignoreDuplicates and --blackListFileName """ - outfile = '/tmp/test_file_filter.bg' + _, outfile = tempfile.mkstemp(suffix=".bg") #for A, B in [(BAMFILE_FILTER1, BAMFILE_FILTER2), (CRAMFILE_FILTER1, CRAMFILE_FILTER2)]: for A, B in [(BAMFILE_FILTER1, BAMFILE_FILTER2)]: args = "-b1 {} -b2 {} -p 1 -o {} -of bedgraph --samFlagInclude 512 " \ diff --git a/pydeeptools/deeptools/test/test_bamPEFragmentSize.py b/pydeeptools/deeptools/test/test_bamPEFragmentSize.py index 92e9287457..f88f99ded7 100644 --- a/pydeeptools/deeptools/test/test_bamPEFragmentSize.py +++ b/pydeeptools/deeptools/test/test_bamPEFragmentSize.py @@ -1,5 +1,6 @@ import os.path from os import unlink +import tempfile from matplotlib.testing.compare import compare_images @@ -12,7 +13,7 @@ def test_bamPEFragmentSize_histogram(): """ Test histogram plot for bamPEFragmentSize """ - outfile = "/tmp/test_histogram.png" + _, outfile = tempfile.mkstemp(suffix=".png") args = "--bamfiles {}/bowtie2_test1.bam --samplesLabel bowtie2_test1.bam --plotFileFormat png --plotTitle Test-Plot --histogram {}".format( ROOT, outfile ).split() @@ -27,8 +28,8 @@ def test_bamPEFragmentSize_fr_sizes(): """ Test fragment length information for bamPEFragmentSize """ - out_lengths = "/tmp/test_raw_frag_lengths.txt" - out_metrics = "/tmp/test_metrics_table.txt" + _, out_lengths = tempfile.mkstemp(suffix=".txt") + _, out_metrics = tempfile.mkstemp(suffix=".txt") args = ( "--bamfiles {}/bowtie2_test1.bam --outRawFragmentLengths {} --table {}".format( ROOT, out_lengths, out_metrics diff --git a/pydeeptools/deeptools/test/test_bigwigAverage.py b/pydeeptools/deeptools/test/test_bigwigAverage.py index 7f86329125..c472b40be7 100644 --- a/pydeeptools/deeptools/test/test_bigwigAverage.py +++ b/pydeeptools/deeptools/test/test_bigwigAverage.py @@ -2,6 +2,7 @@ import os.path from os import unlink +import tempfile ROOT = os.path.dirname(os.path.abspath(__file__)) + "/test_data/" BIGWIG_A = ROOT + "testA_skipNAs.bw" @@ -35,7 +36,7 @@ def test_bigwigAverage(): - outfile = '/tmp/result.bg' + _, outfile = tempfile.mkstemp(suffix=".bg") args = "--bigwigs {} {} -o {} --outFileFormat bedgraph".format(BIGWIG_A, BIGWIG_B, outfile).split() bwAve.main(args) _foo = open(outfile, 'r') @@ -47,7 +48,7 @@ def test_bigwigAverage(): def test_bigwigAverage_skipnas(): - outfile = '/tmp/result.bg' + _, outfile = tempfile.mkstemp(suffix=".bg") args = "--bigwigs {} {} -o {} --skipNAs " \ "--outFileFormat bedgraph".format(BIGWIG_A, BIGWIG_B, outfile).split() bwAve.main(args) @@ -60,7 +61,7 @@ def test_bigwigAverage_skipnas(): def test_bigwigAverageWithScale(): - outfile = '/tmp/result.bg' + _, outfile = tempfile.mkstemp(suffix=".bg") args = "--bigwigs {} {} -o {} --outFileFormat bedgraph --scaleFactors 1:0.5".format(BIGWIG_A, BIGWIG_B, outfile).split() bwAve.main(args) _foo = open(outfile, 'r') @@ -72,7 +73,7 @@ def test_bigwigAverageWithScale(): def test_bigwigAverageThree(): - outfile = '/tmp/result.bg' + _, outfile = tempfile.mkstemp(suffix=".bg") args = "--bigwigs {} {} {} -o {} --outFileFormat bedgraph --scaleFactors 0.75:0.75:.75".format(BIGWIG_A, BIGWIG_A, BIGWIG_B, outfile).split() bwAve.main(args) _foo = open(outfile, 'r') diff --git a/pydeeptools/deeptools/test/test_bigwigCompare_and_multiBigwigSummary.py b/pydeeptools/deeptools/test/test_bigwigCompare_and_multiBigwigSummary.py index 076baa2195..30d222d6f6 100644 --- a/pydeeptools/deeptools/test/test_bigwigCompare_and_multiBigwigSummary.py +++ b/pydeeptools/deeptools/test/test_bigwigCompare_and_multiBigwigSummary.py @@ -5,6 +5,7 @@ import os.path from os import unlink +import tempfile ROOT = os.path.dirname(os.path.abspath(__file__)) + "/test_data/" BIGWIG_A = ROOT + "testA_skipNAs.bw" @@ -38,7 +39,7 @@ def test_bigwigCompare(): - outfile = '/tmp/result.bg' + _, outfile = tempfile.mkstemp(suffix=".bg") args = "-b1 {} -b2 {} -o {} --operation add --outFileFormat bedgraph".format(BIGWIG_A, BIGWIG_B, outfile).split() bwComp.main(args) _foo = open(outfile, 'r') @@ -50,7 +51,7 @@ def test_bigwigCompare(): def test_bigwigCompare_skipnas(): - outfile = '/tmp/result.bg' + _, outfile = tempfile.mkstemp(suffix=".bg") args = "-b1 {} -b2 {} -o {} --operation add --skipNAs " \ "--outFileFormat bedgraph".format(BIGWIG_A, BIGWIG_B, outfile).split() bwComp.main(args) @@ -63,7 +64,7 @@ def test_bigwigCompare_skipnas(): def test_bigwigCompare_skipZeroOverZero(): - outfile = '/tmp/result.bg"' + _, outfile = tempfile.mkstemp(suffix=".bg") args = "-b1 {} -b2 {} -o {} --skipZeroOverZero --pseudocount 1 3 --outFileFormat bedgraph".format(BIGWIG_A, BIGWIG_A, outfile).split() bwComp.main(args) _foo = open(outfile, 'r') @@ -75,7 +76,7 @@ def test_bigwigCompare_skipZeroOverZero(): def test_multiBigwigSummary(): - outfile = '/tmp/result.bg' + _, outfile = tempfile.mkstemp(suffix=".npz") args = "bins -b {} {} --binSize 50 -o {}".format(BIGWIG_A, BIGWIG_B, outfile).split() bwCorr.main(args) resp = np.load(outfile) @@ -93,8 +94,9 @@ def test_multiBigwigSummary_outrawcounts(): """ Test multiBigwigSummary raw counts output """ - outfile = '/tmp/result.bg' - args = "bins -b {} {} --binSize 50 -o /tmp/null --outRawCounts {} ".format(BIGWIG_A, BIGWIG_B, outfile).split() + _, nullfile = tempfile.mkstemp(suffix=".npz") + _, outfile = tempfile.mkstemp(suffix=".txt") + args = "bins -b {} {} --binSize 50 -o {} --outRawCounts {} ".format(BIGWIG_A, BIGWIG_B, nullfile, outfile).split() bwCorr.main(args) _foo = open(outfile, 'r') resp = _foo.read() @@ -107,11 +109,11 @@ def test_multiBigwigSummary_outrawcounts(): """ assert f"{resp}" == f"{expected}", f"{resp} != {expected}" unlink(outfile) - unlink("/tmp/null") + unlink(nullfile) def test_multiBigwigSummary_gtf(): - outfile = '/tmp/_test.npz' + _, outfile = tempfile.mkstemp(suffix=".npz") args = "BED-file -b {0} {0} --BED {1}/test.gtf -o {2}".format(BIGWIG_C, ROOT, outfile).split() bwCorr.main(args) resp = np.load(outfile) @@ -124,7 +126,7 @@ def test_multiBigwigSummary_gtf(): def test_multiBigwigSummary_metagene(): - outfile = '/tmp/_test.npz' + _, outfile = tempfile.mkstemp(suffix=".npz") args = "BED-file --metagene -b {0} {0} --BED {1}/test.gtf -o {2}".format(BIGWIG_C, ROOT, outfile).split() bwCorr.main(args) resp = np.load(outfile) diff --git a/pydeeptools/deeptools/test/test_computeMatrixOperations.py b/pydeeptools/deeptools/test/test_computeMatrixOperations.py index c253431d7d..1f1af32c11 100644 --- a/pydeeptools/deeptools/test/test_computeMatrixOperations.py +++ b/pydeeptools/deeptools/test/test_computeMatrixOperations.py @@ -5,6 +5,7 @@ import hashlib import gzip import json +import tempfile __author__ = 'Devon' @@ -30,7 +31,7 @@ def testSubset(self): """ dCorrect = {"verbose": True, "scale": 1, "skip zeros": False, "nan after end": False, "sort using": "mean", "unscaled 5 prime": [0, 0, 0, 0], "body": [1000, 1000, 1000, 1000], "sample_labels": ["SRR648667.forward", "SRR648668.forward", "SRR648669.forward", "SRR648670.forward"], "downstream": [0, 0, 0, 0], "unscaled 3 prime": [0, 0, 0, 0], "group_labels": ["genes"], "bin size": [10, 10, 10, 10], "upstream": [0, 0, 0, 0], "group_boundaries": [0, 196], "sample_boundaries": [0, 100, 200, 300, 400], "max threshold": None, "ref point": [None, None, None, None], "min threshold": None, "sort regions": "no", "proc number": 20, "bin avg type": "mean", "missing data as zero": False} - oname = "/tmp/subset.mat.gz" + _, oname = tempfile.mkstemp(suffix=".mat.gz") args = "subset -m {} --sample SRR648667.forward SRR648668.forward SRR648669.forward SRR648670.forward -o {}".format(self.matrix, oname) args = args.split() cmo.main(args) @@ -48,7 +49,7 @@ def testRelabel(self): computeMatrixOperations relabel """ dCorrect = {"verbose": True, "scale": 1, "skip zeros": False, "nan after end": False, "sort using": "mean", "unscaled 5 prime": [0, 0, 0, 0, 0, 0, 0, 0], "body": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000], "sample_labels": ["first", "sec ond", "3rd", "4th", "5th", "6th", "7th", "8th"], "downstream": [0, 0, 0, 0, 0, 0, 0, 0], "unscaled 3 prime": [0, 0, 0, 0, 0, 0, 0, 0], "group_labels": ["foo bar"], "bin size": [10, 10, 10, 10, 10, 10, 10, 10], "upstream": [0, 0, 0, 0, 0, 0, 0, 0], "group_boundaries": [0, 196], "sample_boundaries": [0, 100, 200, 300, 400, 500, 600, 700, 800], "max threshold": None, "ref point": [None, None, None, None, None, None, None, None], "min threshold": None, "sort regions": "no", "proc number": 20, "bin avg type": "mean", "missing data as zero": False} - oname = "/tmp/relabeled.mat.gz" + _, oname = tempfile.mkstemp(suffix=".mat.gz") args = "relabel -m {} -o {} --sampleLabels first sec_ond 3rd 4th 5th 6th 7th 8th --groupLabels foo_bar".format(self.matrix, oname) args = args.split() args[7] = 'sec ond' # split mucks up spaces @@ -65,7 +66,7 @@ def testfilterStrand(self): computeMatrixOperations filterStrand """ dCorrect = {"verbose": True, "scale": 1, "skip zeros": False, "nan after end": False, "sort using": "mean", "unscaled 5 prime": [0, 0, 0, 0, 0, 0, 0, 0], "body": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000], "sample_labels": ["SRR648667.forward", "SRR648668.forward", "SRR648669.forward", "SRR648670.forward", "SRR648667.reverse", "SRR648668.reverse", "SRR648669.reverse", "SRR648670.reverse"], "downstream": [0, 0, 0, 0, 0, 0, 0, 0], "unscaled 3 prime": [0, 0, 0, 0, 0, 0, 0, 0], "group_labels": ["genes"], "bin size": [10, 10, 10, 10, 10, 10, 10, 10], "upstream": [0, 0, 0, 0, 0, 0, 0, 0], "group_boundaries": [0, 107], "sample_boundaries": [0, 100, 200, 300, 400, 500, 600, 700, 800], "max threshold": None, "ref point": [None, None, None, None, None, None, None, None], "min threshold": None, "sort regions": "no", "proc number": 20, "bin avg type": "mean", "missing data as zero": False} - oname = "/tmp/filterStrand1.mat.gz" + _, oname = tempfile.mkstemp(suffix=".mat.gz") args = "filterStrand -m {} -o {} --strand +".format(self.matrix, oname) args = args.split(' ') cmo.main(args) @@ -79,7 +80,7 @@ def testfilterStrand(self): os.remove(oname) dCorrect = {u'verbose': True, u'scale': 1, u'skip zeros': False, u'nan after end': False, u'sort using': u'mean', u'unscaled 5 prime': [0, 0, 0, 0, 0, 0, 0, 0], u'body': [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000], u'sample_labels': [u'SRR648667.forward', u'SRR648668.forward', u'SRR648669.forward', u'SRR648670.forward', u'SRR648667.reverse', u'SRR648668.reverse', u'SRR648669.reverse', u'SRR648670.reverse'], u'downstream': [0, 0, 0, 0, 0, 0, 0, 0], u'unscaled 3 prime': [0, 0, 0, 0, 0, 0, 0, 0], u'group_labels': [u'genes'], u'bin size': [10, 10, 10, 10, 10, 10, 10, 10], u'upstream': [0, 0, 0, 0, 0, 0, 0, 0], u'group_boundaries': [0, 89], u'sample_boundaries': [0, 100, 200, 300, 400, 500, 600, 700, 800], u'missing data as zero': False, u'ref point': [None, None, None, None, None, None, None, None], u'min threshold': None, u'sort regions': u'no', u'proc number': 20, u'bin avg type': u'mean', u'max threshold': None} - oname = "/tmp/filterStrand2.mat.gz" + _, oname = tempfile.mkstemp(suffix=".mat.gz") args = "filterStrand -m {} -o {} --strand -".format(self.matrix, oname) args = args.split() cmo.main(args) @@ -97,7 +98,7 @@ def testrbind(self): computeMatrixOperations rbind """ dCorrect = {"verbose": True, "scale": 1, "skip zeros": False, "nan after end": False, "sort using": "mean", "unscaled 5 prime": [0, 0, 0, 0, 0, 0, 0, 0], "body": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000], "sample_labels": ["SRR648667.forward", "SRR648668.forward", "SRR648669.forward", "SRR648670.forward", "SRR648667.reverse", "SRR648668.reverse", "SRR648669.reverse", "SRR648670.reverse"], "downstream": [0, 0, 0, 0, 0, 0, 0, 0], "unscaled 3 prime": [0, 0, 0, 0, 0, 0, 0, 0], "group_labels": ["genes"], "bin size": [10, 10, 10, 10, 10, 10, 10, 10], "upstream": [0, 0, 0, 0, 0, 0, 0, 0], "group_boundaries": [0, 392], "sample_boundaries": [0, 100, 200, 300, 400, 500, 600, 700, 800], "max threshold": None, "ref point": [None, None, None, None, None, None, None, None], "min threshold": None, "sort regions": "no", "proc number": 20, "bin avg type": "mean", "missing data as zero": False} - oname = "/tmp/rbind.mat.gz" + _, oname = tempfile.mkstemp(suffix=".mat.gz") args = "rbind -m {0} {0} -o {1}".format(self.matrix, oname) args = args.split() cmo.main(args) @@ -115,7 +116,7 @@ def testrbind2(self): computeMatrixOperations rbind with different groups """ dCorrect = {"verbose": False, "scale": 1, "skip zeros": False, "nan after end": False, "sort using": "mean", "unscaled 5 prime": [0], "body": [2], "sample_labels": ["signal"], "downstream": [1], "unscaled 3 prime": [0], "group_labels": ["somegenes", "othergenes"], "bin size": [1], "upstream": [1], "group_boundaries": [0, 3, 7], "sample_boundaries": [0, 4], "max threshold": None, "ref point": [None], "min threshold": None, "sort regions": "keep", "proc number": 1, "bin avg type": "mean", "missing data as zero": True} - oname = "/tmp/rbind2.mat.gz" + _, oname = tempfile.mkstemp(suffix=".mat.gz") args = "rbind -m {0} {1} -o {2}".format(self.rbindMatrix1, self.rbindMatrix2, oname) args = args.split() cmo.main(args) @@ -133,7 +134,7 @@ def testcbind(self): computeMatrixOperations cbind """ dCorrect = {"verbose": True, "scale": 1, "skip zeros": False, "nan after end": False, "sort using": "mean", "unscaled 5 prime": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "body": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000], "sample_labels": ["SRR648667.forward", "SRR648668.forward", "SRR648669.forward", "SRR648670.forward", "SRR648667.reverse", "SRR648668.reverse", "SRR648669.reverse", "SRR648670.reverse", "SRR648667.forward", "SRR648668.forward", "SRR648669.forward", "SRR648670.forward", "SRR648667.reverse", "SRR648668.reverse", "SRR648669.reverse", "SRR648670.reverse"], "downstream": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "unscaled 3 prime": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "group_labels": ["genes"], "bin size": [10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10], "upstream": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "group_boundaries": [0, 196], "sample_boundaries": [0, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 1100, 1200, 1300, 1400, 1500, 1600], "max threshold": None, "ref point": [None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None], "min threshold": None, "sort regions": "no", "proc number": 20, "bin avg type": "mean", "missing data as zero": False} - oname = "/tmp/filterStrand.mat.gz" + _, oname = tempfile.mkstemp(suffix=".mat.gz") args = "cbind -m {0} {0} -o {1}".format(self.matrix, oname) args = args.split() cmo.main(args) @@ -151,7 +152,7 @@ def testsort(self): computeMatrixOperations sort """ dCorrect = {"verbose": True, "scale": 1, "skip zeros": False, "nan after end": False, "sort using": "mean", "unscaled 5 prime": [0, 0, 0, 0, 0, 0, 0, 0], "body": [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000], "sample_labels": ["SRR648667.forward", "SRR648668.forward", "SRR648669.forward", "SRR648670.forward", "SRR648667.reverse", "SRR648668.reverse", "SRR648669.reverse", "SRR648670.reverse"], "downstream": [0, 0, 0, 0, 0, 0, 0, 0], "unscaled 3 prime": [0, 0, 0, 0, 0, 0, 0, 0], "group_labels": ["genes"], "bin size": [10, 10, 10, 10, 10, 10, 10, 10], "upstream": [0, 0, 0, 0, 0, 0, 0, 0], "group_boundaries": [0, 196], "sample_boundaries": [0, 100, 200, 300, 400, 500, 600, 700, 800], "max threshold": None, "ref point": [None, None, None, None, None, None, None, None], "min threshold": None, "sort regions": "no", "proc number": 20, "bin avg type": "mean", "missing data as zero": False} - oname = "/tmp/sorted.mat.gz" + _, oname = tempfile.mkstemp(suffix=".mat.gz") args = "sort -m {} -o {} -R {}".format(self.matrix, oname, self.bed) args = args.split() cmo.main(args) diff --git a/pydeeptools/deeptools/test/test_correctGCBias.py b/pydeeptools/deeptools/test/test_correctGCBias.py index 766c4868de..80aad79037 100644 --- a/pydeeptools/deeptools/test/test_correctGCBias.py +++ b/pydeeptools/deeptools/test/test_correctGCBias.py @@ -5,6 +5,7 @@ import os.path from os import unlink import pysam +import tempfile ROOT = os.path.dirname(os.path.abspath(__file__)) + "/test_data/" @@ -16,11 +17,11 @@ def test_correctGCBias(): GCbiasFreq = ROOT + 'computeGCBias_result1.tabular' BAM = ROOT + 'paired_chr2L.bam' GENOME = ROOT + 'sequence.2bit' - outfile = '/tmp/test_correctbias.bam' + _, outfile = tempfile.mkstemp(suffix=".bam") args = "--GCbiasFrequenciesFile {} --bamfile {} --genome {} --effectiveGenomeSize 10050 --correctedFile {}".format(GCbiasFreq, BAM, GENOME, outfile).split() deeptools.correctGCBias.main(args) alignment_count = pysam.AlignmentFile(outfile, "rb").count() expected_count = 11630 assert abs(alignment_count - expected_count) < 50 - unlink(outfile) \ No newline at end of file + unlink(outfile) diff --git a/pydeeptools/deeptools/test/test_data/alignmentSieve.bam b/pydeeptools/deeptools/test/test_data/alignmentSieve.bam deleted file mode 100644 index 560ab32da9..0000000000 Binary files a/pydeeptools/deeptools/test/test_data/alignmentSieve.bam and /dev/null differ diff --git a/pydeeptools/deeptools/test/test_data/alignmentSieve1.bam b/pydeeptools/deeptools/test/test_data/alignmentSieve1.bam new file mode 100644 index 0000000000..d10930ffd5 Binary files /dev/null and b/pydeeptools/deeptools/test/test_data/alignmentSieve1.bam differ diff --git a/pydeeptools/deeptools/test/test_data/alignmentSieve1.filtered.bam b/pydeeptools/deeptools/test/test_data/alignmentSieve1.filtered.bam new file mode 100644 index 0000000000..3a40ae83e7 Binary files /dev/null and b/pydeeptools/deeptools/test/test_data/alignmentSieve1.filtered.bam differ diff --git a/pydeeptools/deeptools/test/test_data/alignmentSieve2.bam b/pydeeptools/deeptools/test/test_data/alignmentSieve2.bam index aebd0399f4..8aaf9a0e32 100644 Binary files a/pydeeptools/deeptools/test/test_data/alignmentSieve2.bam and b/pydeeptools/deeptools/test/test_data/alignmentSieve2.bam differ diff --git a/pydeeptools/deeptools/test/test_data/alignmentSieve3.bam b/pydeeptools/deeptools/test/test_data/alignmentSieve3.bam index e3afc22700..de8b40d24c 100644 Binary files a/pydeeptools/deeptools/test/test_data/alignmentSieve3.bam and b/pydeeptools/deeptools/test/test_data/alignmentSieve3.bam differ diff --git a/pydeeptools/deeptools/test/test_data/alignmentSieve4.bam b/pydeeptools/deeptools/test/test_data/alignmentSieve4.bam new file mode 100644 index 0000000000..7cb306a71c Binary files /dev/null and b/pydeeptools/deeptools/test/test_data/alignmentSieve4.bam differ diff --git a/pydeeptools/deeptools/test/test_data/alignmentSieve5.bam b/pydeeptools/deeptools/test/test_data/alignmentSieve5.bam new file mode 100644 index 0000000000..72c1aff9d5 Binary files /dev/null and b/pydeeptools/deeptools/test/test_data/alignmentSieve5.bam differ diff --git a/pydeeptools/deeptools/test/test_data/alignmentSieve6.bam b/pydeeptools/deeptools/test/test_data/alignmentSieve6.bam new file mode 100644 index 0000000000..c56bbb9ece Binary files /dev/null and b/pydeeptools/deeptools/test/test_data/alignmentSieve6.bam differ diff --git a/pydeeptools/deeptools/test/test_data/alignmentSieve7.bam b/pydeeptools/deeptools/test/test_data/alignmentSieve7.bam new file mode 100644 index 0000000000..ec948f28c9 Binary files /dev/null and b/pydeeptools/deeptools/test/test_data/alignmentSieve7.bam differ diff --git a/pydeeptools/deeptools/test/test_data/alignmentSieve_blacklist.bed b/pydeeptools/deeptools/test/test_data/alignmentSieve_blacklist.bed new file mode 100644 index 0000000000..b1f63508c6 --- /dev/null +++ b/pydeeptools/deeptools/test/test_data/alignmentSieve_blacklist.bed @@ -0,0 +1 @@ +chr2L 5000 5100 diff --git a/pydeeptools/deeptools/test/test_data/bamPEFragmentSize_histogram_result1.png b/pydeeptools/deeptools/test/test_data/bamPEFragmentSize_histogram_result1.png index 8268288a32..fbc0dc7f4c 100644 Binary files a/pydeeptools/deeptools/test/test_data/bamPEFragmentSize_histogram_result1.png and b/pydeeptools/deeptools/test/test_data/bamPEFragmentSize_histogram_result1.png differ diff --git a/pydeeptools/deeptools/test/test_data/paired_chr2L.cram b/pydeeptools/deeptools/test/test_data/paired_chr2L.cram deleted file mode 100644 index 4789b1513f..0000000000 Binary files a/pydeeptools/deeptools/test/test_data/paired_chr2L.cram and /dev/null differ diff --git a/pydeeptools/deeptools/test/test_data/plotCorrelation_result1.png b/pydeeptools/deeptools/test/test_data/plotCorrelation_result1.png index a212f44e5b..85d1fc134c 100644 Binary files a/pydeeptools/deeptools/test/test_data/plotCorrelation_result1.png and b/pydeeptools/deeptools/test/test_data/plotCorrelation_result1.png differ diff --git a/pydeeptools/deeptools/test/test_data/plotCorrelation_result2.png b/pydeeptools/deeptools/test/test_data/plotCorrelation_result2.png index db092d9bde..343ae41ba4 100644 Binary files a/pydeeptools/deeptools/test/test_data/plotCorrelation_result2.png and b/pydeeptools/deeptools/test/test_data/plotCorrelation_result2.png differ diff --git a/pydeeptools/deeptools/test/test_heatmapper.py b/pydeeptools/deeptools/test/test_heatmapper.py index c787214299..e06f95aef4 100644 --- a/pydeeptools/deeptools/test/test_heatmapper.py +++ b/pydeeptools/deeptools/test/test_heatmapper.py @@ -1,5 +1,7 @@ import os import sys +import subprocess +import tempfile import deeptools.computeMatrix2 import json @@ -9,6 +11,19 @@ ROOT = os.path.dirname(os.path.abspath(__file__)) + "/test_heatmapper/" +def _run_compute_and_compare(cmd_args_str, master_file): + _, mat_gz = tempfile.mkstemp(suffix=".mat.gz") + args = cmd_args_str.format(ROOT, mat_gz).split() + deeptools.computeMatrix2.main(args) + subprocess.run(['gunzip', '-f', mat_gz], check=True) + mat = mat_gz[:-3] + try: + assert cmpMatrices(master_file, mat) is True + finally: + if os.path.exists(mat): + os.remove(mat) + + def cmpMatrices(f1, f2): """ The header produced by computeMatrix will be different every time a command is run in python3! @@ -65,99 +80,53 @@ def cmpMatrices(f1, f2): def test_computeMatrix_reference_point(): - args = "reference-point -R {0}/test2.bed -S {0}/test.bw -b 100 -a 100 " \ - "--outFileName /tmp/_test.mat.gz -bs 1 -p 1".format(ROOT).split() - print(args) - deeptools.computeMatrix2.main(args) - os.system('gunzip -f /tmp/_test.mat.gz') - assert cmpMatrices(ROOT + '/master.mat', '/tmp/_test.mat') is True - os.remove('/tmp/_test.mat') + _run_compute_and_compare("reference-point -R {0}/test2.bed -S {0}/test.bw -b 100 -a 100 " \ + "--outFileName {1} -bs 1 -p 1", ROOT + 'master.mat') def test_computeMatrix_reference_point_center(): - args = "reference-point -R {0}/test2.bed -S {0}/test.bw -b 100 -a 100 --referencePoint center " \ - "--outFileName /tmp/_test.mat.gz -bs 1 -p 1".format(ROOT).split() - deeptools.computeMatrix2.main(args) - os.system('gunzip -f /tmp/_test.mat.gz') - assert cmpMatrices(ROOT + '/master_center.mat', '/tmp/_test.mat') is True - os.remove('/tmp/_test.mat') + _run_compute_and_compare("reference-point -R {0}/test2.bed -S {0}/test.bw -b 100 -a 100 --referencePoint center " \ + "--outFileName {1} -bs 1 -p 1", ROOT + 'master_center.mat') def test_computeMatrix_reference_point_tes(): - args = "reference-point -R {0}/test2.bed -S {0}/test.bw -b 100 -a 100 --referencePoint TES " \ - "--outFileName /tmp/_test.mat.gz -bs 1 -p 1".format(ROOT).split() - deeptools.computeMatrix2.main(args) - os.system('gunzip -f /tmp/_test.mat.gz') - assert cmpMatrices(ROOT + '/master_TES.mat', '/tmp/_test.mat') is True - os.remove('/tmp/_test.mat') + _run_compute_and_compare("reference-point -R {0}/test2.bed -S {0}/test.bw -b 100 -a 100 --referencePoint TES " \ + "--outFileName {1} -bs 1 -p 1", ROOT + 'master_TES.mat') def test_computeMatrix_reference_point_missing_data_as_zero(): - args = "reference-point -R {0}/test2.bed -S {0}/test.bw -b 100 -a 100 " \ - "--outFileName /tmp/_test.mat.gz -bs 1 -p 1 --missingDataAsZero".format(ROOT).split() - deeptools.computeMatrix2.main(args) - os.system('gunzip -f /tmp/_test.mat.gz') - assert cmpMatrices(ROOT + '/master_nan_to_zero.mat', '/tmp/_test.mat') is True - os.remove('/tmp/_test.mat') + _run_compute_and_compare("reference-point -R {0}/test2.bed -S {0}/test.bw -b 100 -a 100 " \ + "--outFileName {1} -bs 1 -p 1 --missingDataAsZero", ROOT + 'master_nan_to_zero.mat') def test_computeMatrix_scale_regions(): - args = "scale-regions -R {0}/test2.bed -S {0}/test.bw -b 100 -a 100 -m 100 " \ - "--outFileName /tmp/_test2.mat.gz -bs 1 -p 1".format(ROOT).split() - - deeptools.computeMatrix2.main(args) - os.system('gunzip -f /tmp/_test2.mat.gz') - assert cmpMatrices(ROOT + '/master_scale_reg.mat', '/tmp/_test2.mat') is True - os.remove('/tmp/_test2.mat') + _run_compute_and_compare("scale-regions -R {0}/test2.bed -S {0}/test.bw -b 100 -a 100 -m 100 " \ + "--outFileName {1} -bs 1 -p 1", ROOT + 'master_scale_reg.mat') def test_computeMatrix_multiple_bed(): - args = "reference-point -R {0}/group1.bed {0}/group2.bed -S {0}/test.bw -b 100 -a 100 " \ - "--outFileName /tmp/_test.mat.gz -bs 1 -p 1".format(ROOT).split() - deeptools.computeMatrix2.main(args) - os.system('gunzip -f /tmp/_test.mat.gz') - assert cmpMatrices(ROOT + '/master_multibed.mat', '/tmp/_test.mat') is True - os.remove('/tmp/_test.mat') + _run_compute_and_compare("reference-point -R {0}/group1.bed {0}/group2.bed -S {0}/test.bw -b 100 -a 100 " \ + "--outFileName {1} -bs 1 -p 1", ROOT + 'master_multibed.mat') def test_computeMatrix_region_extend_over_chr_end(): - args = "reference-point -R {0}/group1.bed {0}/group2.bed -S {0}/test.bw -b 100 -a 500 " \ - "--outFileName /tmp/_test.mat.gz -bs 1 -p 1".format(ROOT).split() - print(args) - deeptools.computeMatrix2.main(args) - os.system('gunzip -f /tmp/_test.mat.gz') - assert cmpMatrices(ROOT + '/master_extend_beyond_chr_size.mat', '/tmp/_test.mat') is True - os.remove('/tmp/_test.mat') + _run_compute_and_compare("reference-point -R {0}/group1.bed {0}/group2.bed -S {0}/test.bw -b 100 -a 500 " \ + "--outFileName {1} -bs 1 -p 1", ROOT + 'master_extend_beyond_chr_size.mat') def test_computeMatrix_unscaled(): - args = "scale-regions -S {0}/unscaled.bigWig -R {0}/unscaled.bed -a 300 -b 500 --unscaled5prime 100 --unscaled3prime 50 " \ - "--outFileName /tmp/_test.mat.gz -bs 10 -p 1".format(ROOT).split() - print(args) - deeptools.computeMatrix2.main(args) - os.system('gunzip -f /tmp/_test.mat.gz') - assert cmpMatrices(ROOT + '/master_unscaled.mat', '/tmp/_test.mat') is True - os.remove('/tmp/_test.mat') + _run_compute_and_compare("scale-regions -S {0}/unscaled.bigWig -R {0}/unscaled.bed -a 300 -b 500 --unscaled5prime 100 --unscaled3prime 50 " \ + "--outFileName {1} -bs 10 -p 1", ROOT + 'master_unscaled.mat') def test_computeMatrix_gtf(): - args = "scale-regions -S {0}../test_data/test1.bw.bw -R {0}../test_data/test.gtf -a 300 -b 500 --unscaled5prime 20 --unscaled3prime 50 " \ - "--outFileName /tmp/_test_gtf.mat.gz -bs 10 -p 1".format(ROOT).split() - print(args) - deeptools.computeMatrix2.main(args) - os.system('gunzip -f /tmp/_test_gtf.mat.gz') - assert cmpMatrices(ROOT + '/master_gtf.mat', '/tmp/_test_gtf.mat') is True - os.remove('/tmp/_test_gtf.mat') + _run_compute_and_compare("scale-regions -S {0}../test_data/test1.bw.bw -R {0}../test_data/test.gtf -a 300 -b 500 --unscaled5prime 20 --unscaled3prime 50 " \ + "--outFileName {1} -bs 10 -p 1", ROOT + 'master_gtf.mat') def test_computeMatrix_metagene(): - args = "scale-regions -S {0}../test_data/test1.bw.bw -R {0}../test_data/test.gtf -a 300 -b 500 --unscaled5prime 20 --unscaled3prime 50 " \ - "--outFileName /tmp/_test_metagene.mat.gz -bs 10 -p 1 --metagene".format(ROOT).split() - print(args) - deeptools.computeMatrix2.main(args) - os.system('gunzip -f /tmp/_test_metagene.mat.gz') - assert cmpMatrices(ROOT + '/master_metagene.mat', '/tmp/_test_metagene.mat') is True - os.remove('/tmp/_test_metagene.mat') + _run_compute_and_compare("scale-regions -S {0}../test_data/test1.bw.bw -R {0}../test_data/test.gtf -a 300 -b 500 --unscaled5prime 20 --unscaled3prime 50 " \ + "--outFileName {1} -bs 10 -p 1 --metagene", ROOT + 'master_metagene.mat') # def test_chopRegions_body(): diff --git a/pydeeptools/deeptools/test/test_multiBamSummary.py b/pydeeptools/deeptools/test/test_multiBamSummary.py index 253a583735..e5ad496284 100644 --- a/pydeeptools/deeptools/test/test_multiBamSummary.py +++ b/pydeeptools/deeptools/test/test_multiBamSummary.py @@ -4,6 +4,7 @@ import os.path from os import unlink +import tempfile ROOT = os.path.dirname(os.path.abspath(__file__)) + "/test_data/" BAM = ROOT + "test1.bam" @@ -14,7 +15,7 @@ def test_multiBamSummary_gtf(): - outfile = '/tmp/_test.npz' + _, outfile = tempfile.mkstemp(suffix=".npz") #for fname in [BAM, CRAM]: for fname in [BAM]: args = 'BED-file --BED {0} -b {1} {1} -o {2}'.format(GTF, fname, outfile).split() @@ -28,7 +29,7 @@ def test_multiBamSummary_gtf(): def test_multiBamSummary_metagene(): - outfile = '/tmp/_test.npz' + _, outfile = tempfile.mkstemp(suffix=".npz") #for fname in [BAM, CRAM]: for fname in [BAM]: args = 'BED-file --BED {0} -b {1} {1} -o {2} --metagene'.format(GTF, fname, outfile).split() @@ -42,8 +43,8 @@ def test_multiBamSummary_metagene(): def test_multiBamSummary_scalingFactors(): - outfile = '/tmp/test.scalingFactors.txt' - outfile2 = '/tmp/_test.npz' + _, outfile = tempfile.mkstemp(suffix=".txt") + _, outfile2 = tempfile.mkstemp(suffix=".npz") args = 'bins --binSize 50 -b {} {} --scalingFactors {} -o {} --verbose'.format(BAMA, BAMB, outfile, outfile2).split() mbs.main(args) resp = open(outfile).read().strip().split('\n') diff --git a/pydeeptools/deeptools/test/test_plotCoverage/plotCoverage_default.png b/pydeeptools/deeptools/test/test_plotCoverage/plotCoverage_default.png index f0f57324ef..8e8e8b1c58 100644 Binary files a/pydeeptools/deeptools/test/test_plotCoverage/plotCoverage_default.png and b/pydeeptools/deeptools/test/test_plotCoverage/plotCoverage_default.png differ diff --git a/pydeeptools/deeptools/test/test_plotEnrichment/plotEnrichment_defaults.png b/pydeeptools/deeptools/test/test_plotEnrichment/plotEnrichment_defaults.png index 6f0f6f3caf..e379baf838 100644 Binary files a/pydeeptools/deeptools/test/test_plotEnrichment/plotEnrichment_defaults.png and b/pydeeptools/deeptools/test/test_plotEnrichment/plotEnrichment_defaults.png differ diff --git a/pydeeptools/deeptools/test/test_plotFingerprint/test_plotFingerprint_default.png b/pydeeptools/deeptools/test/test_plotFingerprint/test_plotFingerprint_default.png index 78824da9fe..275b8087c3 100644 Binary files a/pydeeptools/deeptools/test/test_plotFingerprint/test_plotFingerprint_default.png and b/pydeeptools/deeptools/test/test_plotFingerprint/test_plotFingerprint_default.png differ diff --git a/pydeeptools/deeptools/test/test_plotHeatmap/plotHeatmap_default.png b/pydeeptools/deeptools/test/test_plotHeatmap/plotHeatmap_default.png index 606c1b4e22..726fdc1f3d 100644 Binary files a/pydeeptools/deeptools/test/test_plotHeatmap/plotHeatmap_default.png and b/pydeeptools/deeptools/test/test_plotHeatmap/plotHeatmap_default.png differ diff --git a/pydeeptools/deeptools/test/test_plotPCA/test_plotPCA_default.png b/pydeeptools/deeptools/test/test_plotPCA/test_plotPCA_default.png index ec90f17dfb..61833f0da7 100644 Binary files a/pydeeptools/deeptools/test/test_plotPCA/test_plotPCA_default.png and b/pydeeptools/deeptools/test/test_plotPCA/test_plotPCA_default.png differ diff --git a/pydeeptools/deeptools/test/test_plotPCA/test_plotPCA_default.tsv b/pydeeptools/deeptools/test/test_plotPCA/test_plotPCA_default.tsv index 818ee46c6d..2825477a63 100644 --- a/pydeeptools/deeptools/test/test_plotPCA/test_plotPCA_default.tsv +++ b/pydeeptools/deeptools/test/test_plotPCA/test_plotPCA_default.tsv @@ -1,7 +1,7 @@ Component wt1 wt2 wt3 kd1 kd2 kd3 Eigenvalue -1 -1.9593953629718786 -0.05992632052266235 0.01551443898050737 0.36087262695399747 -0.06417725736658209 0.012713770441420403 5.807692278755936 -2 -3.359410695125286 0.14539309680293142 -0.13474864281257853 -0.01241116566407577 -0.12363598259527717 -0.060663076611993313 0.07423028883557825 -3 -2.335822964750958 -0.09987722598713865 -0.034135419740642575 -0.07165342296216214 0.051703858136008 0.13732570368291072 0.04897177773493676 -4 -0.3078978392240127 -0.004185214180292649 -0.013424388619809194 0.05146245577652413 -0.0794287737098722 0.07458644503209262 0.03680941552538939 -5 -1.280108686154136 0.07795301634249859 -0.047687205636145584 -0.023214274278218 -0.03240567700055615 0.0742066588868442 0.026706723301448194 -6 -3.1274153845376063 -0.17202014078313513 0.08493798982333466 0.014293535309782638 0.018508562924869645 0.1575339674343244 0.017613563942900697 +1 -1.9593953629718786 -0.0599263205226625 0.015514438980507314 0.3608726269539977 -0.06417725736658203 0.012713770441420292 5.807692278755936 +2 -3.359410695125286 0.1453930968029311 -0.13474864281257853 -0.012411165664075682 -0.12363598259527717 -0.060663076611993313 0.07423028883557825 +3 -2.335822964750958 -0.0998772259871388 -0.03413541974064263 -0.07165342296216205 0.05170385813600817 0.13732570368291072 0.04897177773493676 +4 -0.3078978392240127 -0.004185214180292667 -0.013424388619809201 0.051462455776524134 -0.0794287737098722 0.07458644503209262 0.03680941552538939 +5 -1.280108686154136 0.0779530163424985 -0.04768720563614561 -0.023214274278217935 -0.0324056770005561 0.0742066588868442 0.026706723301448194 +6 -3.1274153845376063 -0.17202014078313563 0.08493798982333454 0.014293535309782737 0.01850856292486998 0.1575339674343244 0.017613563942900697 diff --git a/pydeeptools/deeptools/test/test_plotProfile/plotProfile_default.png b/pydeeptools/deeptools/test/test_plotProfile/plotProfile_default.png index f7ba808a38..7308a61d14 100644 Binary files a/pydeeptools/deeptools/test/test_plotProfile/plotProfile_default.png and b/pydeeptools/deeptools/test/test_plotProfile/plotProfile_default.png differ diff --git a/pydeeptools/deeptools/test/test_plotcorrelation.py b/pydeeptools/deeptools/test/test_plotcorrelation.py index 27a7415ecd..0d3f78c978 100644 --- a/pydeeptools/deeptools/test/test_plotcorrelation.py +++ b/pydeeptools/deeptools/test/test_plotcorrelation.py @@ -3,7 +3,7 @@ import os.path from os import unlink from matplotlib.testing.compare import compare_images - +import tempfile ROOT = os.path.dirname(os.path.abspath(__file__)) + "/test_data/" COR_DATA_IN1 = ROOT + "multiBamSummary_result1.npz" @@ -16,8 +16,8 @@ def test_correlation_plot_with_minimal_options(): Test minimal command line args for correlation plot with output as correlation matrix along with matrix """ - out_matrix = '/tmp/correlation_matrix.tsv' - out_png = '/tmp/correlation_plot1.png' + _, out_matrix = tempfile.mkstemp(suffix=".tsv") + _, out_png = tempfile.mkstemp(suffix=".png") args = "--corData {} -p heatmap -c pearson -o {} --outFileCorMatrix {}".format(COR_DATA_IN1, out_png, out_matrix).split() pc.main(args) @@ -29,17 +29,14 @@ def test_correlation_plot_with_minimal_options(): res = compare_images(COR_PLOT_1, out_png, 50) assert res is None, "Plots do not match" - unlink(out_png) - def test_correlation_plot_scatter(): """ Test command line args for correlation plot with output as scatter plot """ - out_png2 = '/tmp/correlation_plot2.png' + _, out_png2 = tempfile.mkstemp(suffix=".png") args = "--corData {} -p scatterplot -c pearson -o {}".format(COR_DATA_IN1, out_png2).split() pc.main(args) res = compare_images(COR_PLOT_2, out_png2, 50) assert res is None, "Plots do not match" - unlink(out_png2) diff --git a/pydeeptools/deeptools/test/test_readFiltering.py b/pydeeptools/deeptools/test/test_readFiltering.py index 8227530fbd..6c1486f386 100644 --- a/pydeeptools/deeptools/test/test_readFiltering.py +++ b/pydeeptools/deeptools/test/test_readFiltering.py @@ -4,6 +4,7 @@ from os import unlink import hashlib import pysam +import tempfile ROOT = os.path.dirname(os.path.abspath(__file__)) + "/test_data/" @@ -16,7 +17,7 @@ def test_estimate_read_filtering_minimal(): """ Minimal testing """ - outfile = '/tmp/test_minimal.txt' + _, outfile = tempfile.mkstemp(suffix=".txt") args = '-b {} -o {}'.format(BAMFILE_FILTER, outfile).split() est.main(args) @@ -37,7 +38,7 @@ def test_estimate_read_filtering_params(): """ --minMappingQuality 10 --samFlagExclude 512 --ignoreDuplicates -bl """ - outfile = '/tmp/test_params.txt' + _, outfile = tempfile.mkstemp(suffix=".txt") args = '-b {} --minMappingQuality 10 --samFlagExclude 512 --ignoreDuplicates -bl {} -o {}'.format(BAMFILE_FILTER, BEDFILE_FILTER, outfile).split() est.main(args) @@ -58,9 +59,9 @@ def test_sieve(): """ Test filtering a BAM file by MAPQ, flag, and blacklist """ - outfile = '/tmp/test_sieve.bam' - outfiltered = '/tmp/test_sieveFiltered.bam' - outlog = '/tmp/test_sieve.log' + _, outfile = tempfile.mkstemp(suffix=".bam") + _, outfiltered = tempfile.mkstemp(suffix=".bam") + _, outlog = tempfile.mkstemp(suffix=".log") args = '-b {} --smartLabels --minMappingQuality 10 --samFlagExclude 512 -bl {} -o {} --filterMetrics {} --filteredOutReads {}'.format(BAMFILE_FILTER, BEDFILE_FILTER, outfile, outlog, outfiltered).split() sieve.main(args) @@ -88,7 +89,7 @@ def test_sieve_BED(): """ Test alignmentSieve with the --BED option """ - outfile = '/tmp/test_sieve.bed' + _, outfile = tempfile.mkstemp(suffix=".bed") args = '-b {} --minMappingQuality 10 --BED -o {}'.format(PAIREDBAMFILE_FILTER, outfile).split() sieve.main(args) @@ -130,7 +131,7 @@ def test_sieve_BED_shift(): """ Test alignmentSieve --BED --shift """ - outfile = '/tmp/test_sieve_shift.bed' + _, outfile = tempfile.mkstemp(suffix=".bed") args = '-b {} --minMappingQuality 10 --BED -o {} --shift 1 -2 3 -4'.format(PAIREDBAMFILE_FILTER, outfile).split() sieve.main(args) diff --git a/pyproject.toml b/pyproject.toml index 5a23c240e0..e8e5810905 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,7 +62,7 @@ repository = "https://github.com/deeptools/deepTools" packages = ["deeptools"] [project.scripts] -alignmentSieve = "deeptools.alignmentSieve:main" +alignmentSieve_old = "deeptools.alignmentSieve:main" #bamCompare_old = "deeptools.bamCompare:main" #bamCoverage_old = "deeptools.bamCoverage:main" bamPEFragmentSize = "deeptools.bamPEFragmentSize:main" @@ -87,7 +87,7 @@ plotProfile = "deeptools.plotProfile:main" bamCoverage = "deeptools.bamCoverage2:main" bamCompare = "deeptools.bamCompare2:main" computeMatrix = "deeptools.computeMatrix2:main" -#alignmentSieve = "deeptools.alignmentSieve2:main" +alignmentSieve = "deeptools.alignmentSieve2:main" multiBamSummary = "deeptools.multiBamSummary2:main" [tool.pixi.workspace] diff --git a/src/alignmentsieve.rs b/src/alignmentsieve.rs index d680c8eb21..9ebad10fdd 100644 --- a/src/alignmentsieve.rs +++ b/src/alignmentsieve.rs @@ -1,14 +1,13 @@ -use crate::covcalc::{parse_regions, Region}; use crate::filehandler::{is_bed_or_gtf, read_bedfile}; use crate::filtering::Alignmentfilters; use pyo3::prelude::*; use pyo3::types::PyList; -use rayon::prelude::*; -use rayon::ThreadPoolBuilder; -use rust_htslib::bam::{self, Header, IndexedReader, Read, Reader, Writer}; +use rust_htslib::bam::record::CigarString; +use rust_htslib::bam::{self, Header, HeaderView, Read, Reader, Writer}; +use std::collections::HashMap; use std::fs::File; -use std::io::Write; -use tempfile::{Builder, TempPath}; +use std::io::{BufWriter, Write}; +use std::path::Path; #[pyfunction] pub fn r_alignmentsieve( @@ -20,7 +19,7 @@ pub fn r_alignmentsieve( filtered_out_readsfile: &str, // filtered_out_reads bam/bedfile. verbose: bool, // verbose shift: Py, // python list of the shift to perform. - _bed: bool, // output format in BEDPE. + bed: bool, // output format in BEDPE. filterrnastrand: &str, // "forward", "reverse" or "None". minmappingquality: u8, // minimum mapping quality. samflaginclude: u16, // sam flag include @@ -28,41 +27,50 @@ pub fn r_alignmentsieve( blacklist: &str, // blacklist file name. minfraglen: u32, // minimum fragment length. maxfraglen: u32, // maximum fragment length. - _extend_reads: u32, - _center_reads: bool, + label: &str, // user defined label + smartlabels: bool, // derive label from filename ) -> PyResult<()> { - // Input bam file - let bam = Reader::from_path(bamifile).unwrap(); + // Open input BAM once and stream through in order + let mut bam = Reader::from_path(bamifile).unwrap(); let header = Header::from_template(bam.header()); - let _header_view = bam.header().clone(); + let header_view = bam.header().clone(); - let mut write_filters: bool = false; - if filtered_out_readsfile != "None" { - write_filters = true; + if verbose { + println!("Opening BAM file: {}", bamifile); } + + let write_filters = filtered_out_readsfile != "None"; let readshift: Vec = shift.extract(py).expect("Failed to extract shift"); // shift is of length 0, 2, or 4. - // Define regions - let (regions, chromsizes) = parse_regions("None", vec![bamifile]); - // If there is a blacklist, read it. - let mut backlistregions: Option> = None; - if blacklist != "None" { - // Check if it's a bed or gtf file + // Build chrom_sizes from BAM header + let chrom_sizes: HashMap = (0..header_view.target_count()) + .map(|tid| { + ( + String::from_utf8(header_view.tid2name(tid).to_vec()).unwrap(), + header_view.target_len(tid).unwrap() as u64, + ) + }) + .collect(); + + // Load blacklist regions if provided + let blacklist_regions = if blacklist != "None" { let isbed = is_bed_or_gtf(blacklist); + let chrom_keys: Vec<&String> = chrom_sizes.keys().collect(); match isbed.as_str() { "gtf" => panic!("Error: Please provide a bed file for the blacklist."), "bed" => { - let (bls, _) = - read_bedfile(&blacklist.to_string(), false, chromsizes.keys().collect()); - backlistregions = Some(bls); + let (bls, _) = read_bedfile(&blacklist.to_string(), false, chrom_keys); + Some(bls) } _ => panic!("Error: Cannot determine filetype of blacklist file."), } - } + } else { + None + }; let filters = Alignmentfilters::new( - backlistregions, + blacklist_regions, Some(minmappingquality), Some(samflaginclude), Some(samflagexclude), @@ -75,283 +83,397 @@ pub fn r_alignmentsieve( None, None, ); - let pool = ThreadPoolBuilder::new().num_threads(1).build().unwrap(); - let (sieve, filtersieve, totalreads, filteredreads) = pool.install(|| { - regions - .par_iter() - .map(|i| { - sieve_bamregion( - bamifile, - i, - &filters, - &readshift, - write_filters, - nproc, - verbose, - ) - }) - .reduce( - || (Vec::new(), Vec::new(), 0, 0), - |(mut _sieve, mut _filtersieve, mut _total, mut _filter), - (sieve, filtersieve, total, filter)| { - _sieve.extend(sieve); - _filtersieve.extend(filtersieve); - _total += total; - _filter += filter; - (_sieve, _filtersieve, _total, _filter) - }, - ) - }); - - // write output - let mut obam = Writer::from_path(ofile, &header, bam::Format::Bam).unwrap(); - let _ = obam.set_threads(nproc); - for sb in sieve.into_iter() { - if let Some(sb) = sb { - let mut bam = Reader::from_path(&sb).unwrap(); - for result in bam.records() { - let record = result.unwrap(); - obam.write(&record).unwrap(); + + // Open output writers + let mut obam = if !bed { + Some(Writer::from_path(ofile, &header, bam::Format::Bam).unwrap()) + } else { + None + }; + if let Some(ref mut w) = obam { + let _ = w.set_threads(nproc); + } + let mut obed = if bed { + Some(BufWriter::new(File::create(ofile).unwrap())) + } else { + None + }; + + let mut ofilterbam = if write_filters && !bed { + Some(Writer::from_path(filtered_out_readsfile, &header, bam::Format::Bam).unwrap()) + } else { + None + }; + if let Some(ref mut w) = ofilterbam { + let _ = w.set_threads(nproc); + } + let mut ofilterbed = if write_filters && bed { + Some(BufWriter::new( + File::create(filtered_out_readsfile).unwrap(), + )) + } else { + None + }; + + let mut total_reads: u64 = 0; + let mut filtered_reads: u64 = 0; + + for result in bam.records() { + let record = result.unwrap(); + total_reads += 1; + + let filtered = filter_record(&record, &header_view, &filters); + + if filtered { + filtered_reads += 1; + if let Some(ref mut w) = ofilterbam { + w.write(&record).unwrap(); + } else if let Some(ref mut bw) = ofilterbed { + write_bed_line(&record, &header_view, &chrom_sizes, bw); } + continue; } - } - // write filtered reads if necessary - if write_filters { - let mut ofilterbam = - Writer::from_path(filtered_out_readsfile, &header, bam::Format::Bam).unwrap(); - let _ = ofilterbam.set_threads(nproc); - for sb in filtersieve.into_iter() { - if let Some(sb) = sb { - let mut bam = Reader::from_path(&sb).unwrap(); - for result in bam.records() { - let record = result.unwrap(); - ofilterbam.write(&record).unwrap(); + + if !readshift.is_empty() { + let chrom_name = + String::from_utf8(header_view.tid2name(record.tid() as u32).to_vec()).unwrap(); + if let Some(shifted) = + apply_shift(&record, &readshift, chrom_sizes.get(&chrom_name).copied()) + { + if let Some(ref mut w) = obam { + w.write(&shifted).unwrap(); + } else if let Some(ref mut bw) = obed { + write_bed_line(&shifted, &header_view, &chrom_sizes, bw); } } + continue; + } + + // No shift — write original record + if let Some(ref mut w) = obam { + w.write(&record).unwrap(); + } else if let Some(ref mut bw) = obed { + write_bed_line(&record, &header_view, &chrom_sizes, bw); } } - let _ofilterbam = Writer::from_path(filtered_out_readsfile, &header, bam::Format::Bam).unwrap(); + // Flush writers + if let Some(mut w) = obed { + w.flush().unwrap(); + } + if let Some(mut w) = ofilterbed { + w.flush().unwrap(); + } + // Write filter metrics if filter_metrics != "None" { + let sample_name = if smartlabels { + smart_label(bamifile) + } else if label != "None" { + label.to_string() + } else { + bamifile.to_string() + }; + let mut of = File::create(filter_metrics).unwrap(); - // write header writeln!(of, "#bamFilterReads --filterMetrics").unwrap(); - writeln!(of, "#File\tReads\tRemaining Total\tInitial Reads").unwrap(); + writeln!(of, "#File\tReads Remaining\tTotal Initial Reads").unwrap(); writeln!( of, "{}\t{}\t{}", - bamifile, - totalreads - filteredreads, - totalreads + sample_name, + total_reads - filtered_reads, + total_reads ) .unwrap(); } + if verbose { + println!( + "Total reads: {}, Filtered: {}, Remaining: {}", + total_reads, + filtered_reads, + total_reads - filtered_reads + ); + } + Ok(()) } -fn sieve_bamregion( - ibam: &str, - regstruct: &Region, +/// Determines whether a record should be filtered out. +/// Returns true if the record fails any filter criterion. +fn filter_record( + record: &bam::Record, + bam_header: &HeaderView, alfilters: &Alignmentfilters, - _shift: &Vec, - write_filters: bool, - nproc: usize, - verbose: bool, -) -> (Vec>, Vec>, u64, u64) { - let region = ( - regstruct.chrom.clone(), - regstruct.get_startu(), - regstruct.get_endu(), - ); - let mut total_reads: u64 = 0; - let mut filtered_reads: u64 = 0; - let mut bam = IndexedReader::from_path(ibam).unwrap(); - let header = Header::from_template(bam.header()); +) -> bool { + // Unmapped reads + if record.is_unmapped() { + return true; + } - let mut written = false; - let mut filterwritten = false; - - let sievebam = Builder::new() - .prefix("deeptoolstmp_alsieve_") - .suffix(".bam") - .rand_bytes(12) - .tempfile() - .expect("Failed to create temporary file."); - - let sievebam_path = sievebam.into_temp_path(); - let mut sievebamout = Writer::from_path(&sievebam_path, &header, bam::Format::Bam).unwrap(); - - let filterbam = Builder::new() - .prefix("deeptoolstmp_alsieve_filtered_") - .suffix(".bam") - .rand_bytes(12) - .tempfile() - .expect("Failed to create temporary file."); - let filterbam_path = filterbam.into_temp_path(); - let mut filterbamout = if write_filters { - Some(Writer::from_path(&filterbam_path, &header, bam::Format::Bam).unwrap()) - } else { - None - }; - if nproc > 4 { - let readthreads = 2; - let writethreads = nproc - 2; - let _ = bam.set_threads(readthreads); - let _ = sievebamout.set_threads(writethreads); - if verbose { - println!("Reading = {}, Writing = {}", readthreads, writethreads); - } + // Mapping quality + if record.mapq() < alfilters.minmappingquality { + return true; } - bam.fetch((region.0.as_str(), region.1, region.2)).unwrap(); + // SAM flag include + if alfilters.samflaginclude != 0 + && (record.flags() & alfilters.samflaginclude) != alfilters.samflaginclude + { + return true; + } - for result in bam.records() { - let record = result.unwrap(); - total_reads += 1; + // SAM flag exclude + if alfilters.samflagexclude != 0 && (record.flags() & alfilters.samflagexclude) != 0 { + return true; + } - // Filter reads - // Filter unmapped reads. - if record.is_unmapped() { - filtered_reads += 1; - if let Some(filterbamout) = &mut filterbamout { - filterbamout.write(&record).unwrap(); - filterwritten = true; - } - continue; + // Fragment length — check min and max independently + // minFragmentLength > 0 and tLen < min; maxFragmentLength > 0 and tLen > max) + if record.is_paired() { + let tlen = record.insert_size().abs(); + if alfilters.minfraglen != 0 && tlen < alfilters.minfraglen as i64 { + return true; } - // Mapping qualities. - if record.mapq() < alfilters.minmappingquality { - filtered_reads += 1; - if let Some(filterbamout) = &mut filterbamout { - filterbamout.write(&record).unwrap(); - filterwritten = true; - } - continue; + if alfilters.maxfraglen != 0 && tlen > alfilters.maxfraglen as i64 { + return true; } - - // SAM flags - if alfilters.samflaginclude != 0 && (record.flags() & alfilters.samflaginclude) == 0 { - filtered_reads += 1; - if let Some(filterbamout) = &mut filterbamout { - filterbamout.write(&record).unwrap(); - filterwritten = true; + } else { + let mut tlen: u32 = 0; + for cig in record.cigar().iter() { + match cig { + bam::record::Cigar::Match(len) + | bam::record::Cigar::Del(len) + | bam::record::Cigar::Equal(len) + | bam::record::Cigar::Diff(len) => tlen += len, + _ => (), } - continue; } - if alfilters.samflagexclude != 0 && (record.flags() & alfilters.samflagexclude) != 0 { - filtered_reads += 1; - if let Some(filterbamout) = &mut filterbamout { - filterbamout.write(&record).unwrap(); - filterwritten = true; - } - continue; + if alfilters.minfraglen != 0 && tlen < alfilters.minfraglen { + return true; + } + if alfilters.maxfraglen != 0 && tlen > alfilters.maxfraglen { + return true; } + } - // fragment length - if alfilters.minfraglen != 0 || alfilters.maxfraglen != 0 { - if record.is_paired() { - if record.insert_size().abs() < alfilters.minfraglen as i64 - || record.insert_size().abs() > alfilters.maxfraglen as i64 - { - filtered_reads += 1; - if let Some(filterbamout) = &mut filterbamout { - filterbamout.write(&record).unwrap(); - filterwritten = true; - } - continue; - } - } else { - // Parse cigartuples - let mut tlen: u32 = 0; - for cig in record.cigar().iter() { - match cig { - bam::record::Cigar::Match(len) => tlen += len, - bam::record::Cigar::Del(len) => tlen += len, - bam::record::Cigar::Equal(len) => tlen += len, - bam::record::Cigar::Diff(len) => tlen += len, - _ => (), - } - } - if tlen < alfilters.minfraglen || tlen > alfilters.maxfraglen { - filtered_reads += 1; - if let Some(filterbamout) = &mut filterbamout { - filterbamout.write(&record).unwrap(); - filterwritten = true; - } - continue; + // RNA strand filter + if alfilters.filterrnastrand.as_str() != "None" { + match (alfilters.filterrnastrand.as_str(), record.is_paired()) { + ("forward", true) => { + if !((record.flags() & 144 == 128) || (record.flags() & 96 == 64)) { + return true; } } - } - if alfilters.filterrnastrand.as_str() != "None" { - match (alfilters.filterrnastrand.as_str(), record.is_paired()) { - ("forward", true) => { - if !((record.flags() & 144 == 128) || (record.flags() & 96 == 64)) { - filtered_reads += 1; - if let Some(filterbamout) = &mut filterbamout { - filterbamout.write(&record).unwrap(); - filterwritten = true; - } - continue; - } + ("forward", false) => { + if !(record.flags() & 16 == 16) { + return true; } - ("forward", false) => { - if !(record.flags() & 16 == 16) { - filtered_reads += 1; - if let Some(filterbamout) = &mut filterbamout { - filterbamout.write(&record).unwrap(); - filterwritten = true; - } - continue; - } - } - ("reverse", true) => { - if !((record.flags() & 144 == 144) || (record.flags() & 96 == 96)) { - filtered_reads += 1; - if let Some(filterbamout) = &mut filterbamout { - filterbamout.write(&record).unwrap(); - filterwritten = true; - } - continue; - } + } + ("reverse", true) => { + if !((record.flags() & 144 == 144) || (record.flags() & 96 == 96)) { + return true; } - ("reverse", false) => { - if !(record.flags() & 16 == 0) { - filtered_reads += 1; - if let Some(filterbamout) = &mut filterbamout { - filterbamout.write(&record).unwrap(); - filterwritten = true; - } - continue; - } + } + ("reverse", false) => { + if !(record.flags() & 16 == 0) { + return true; } - _ => {} } + _ => {} } - sievebamout.write(&record).unwrap(); - written = true; } - match (written, filterwritten) { - (true, true) => ( - vec![Some(sievebam_path)], - vec![Some(filterbam_path)], - total_reads, - filtered_reads, - ), - (true, false) => ( - vec![Some(sievebam_path)], - vec![None], - total_reads, - filtered_reads, - ), - (false, true) => ( - vec![None], - vec![Some(filterbam_path)], - total_reads, - filtered_reads, - ), - (false, false) => (vec![None], vec![None], total_reads, filtered_reads), + // Blacklist filter + if alfilters.blacklist.is_some() { + let chrom_name = + String::from_utf8(bam_header.tid2name(record.tid() as u32).to_vec()).unwrap(); + if alfilters.rec_in_blacklist(record, &chrom_name) { + return true; + } + } + + false +} + +/// Writes a single fragment as a BEDPE line (chrom\tstart\tend) if tlen > 0. +fn write_bed_line( + record: &bam::Record, + bam_header: &HeaderView, + chrom_sizes: &HashMap, + bw: &mut BufWriter, +) { + let tlen: i64 = if record.insert_size().abs() > 0 { + record.insert_size() + } else { + let mut cilen: i64 = 0; + for cig in record.cigar().iter() { + match cig { + bam::record::Cigar::Match(len) + | bam::record::Cigar::Del(len) + | bam::record::Cigar::Equal(len) + | bam::record::Cigar::Diff(len) => cilen += *len as i64, + _ => (), + } + } + cilen + }; + + // Only emit when tlen > 0 (keeps exactly one record per fragment for PE reads) + if tlen <= 0 { + return; + } + + let chrom = String::from_utf8(bam_header.tid2name(record.tid() as u32).to_vec()).unwrap(); + let start = record.pos() as i64; + let mut end = start + tlen; + + if let Some(&chrom_len) = chrom_sizes.get(&chrom) { + if end > chrom_len as i64 { + end = chrom_len as i64; + } + } + + if end - start < 1 { + return; + } + + let _ = writeln!(bw, "{}\t{}\t{}", chrom, start, end); +} + +// Calculate query_alignment_end: the 0-based end position of the alignment in the query, +// + excluding trailing soft-clips. +fn query_alignment_end(record: &bam::Record) -> i64 { + let cigar_ops: Vec = record.cigar().iter().cloned().collect(); + let mut read_pos: i64 = 0; + for op in &cigar_ops { + match op { + bam::record::Cigar::Match(l) + | bam::record::Cigar::Ins(l) + | bam::record::Cigar::Equal(l) + | bam::record::Cigar::Diff(l) + | bam::record::Cigar::SoftClip(l) => read_pos += *l as i64, + _ => (), + } + } + // Remove trailing soft-clips + for op in cigar_ops.iter().rev() { + if let bam::record::Cigar::SoftClip(l) = op { + read_pos -= *l as i64; + } else { + break; + } + } + read_pos +} + +fn apply_shift(record: &bam::Record, shift: &[i32], chrom_len: Option) -> Option { + if !record.is_proper_pair() { + return None; + } + if shift.len() < 4 { + return None; + } + + let tlen = record.insert_size(); + let mut start: i64 = record.pos(); + // end = start + b.query_alignment_end + let mut end: i64 = start + query_alignment_end(record); + + // is_first_in_template: true = read1, false = read2 + let is_read1 = record.is_first_in_template(); + let is_read2 = !is_read1; + let is_rev = record.is_reverse(); + + let delta_tlen: i64; + if is_rev && is_read1 { + end -= shift[2] as i64; + delta_tlen = shift[3] as i64 - shift[2] as i64; + } else if is_rev && is_read2 { + end += shift[1] as i64; + delta_tlen = shift[1] as i64 - shift[0] as i64; + } else if !is_rev && is_read1 { + start += shift[0] as i64; + delta_tlen = shift[1] as i64 - shift[0] as i64; + } else { + start -= shift[3] as i64; + delta_tlen = shift[3] as i64 - shift[2] as i64; + } + + // Sanity check: if end - start < 1 + if end - start < 1 { + if is_rev { + start = end - 1; + } else { + end = start + 1; + } + } + if start < 0 { + start = 0; + } + if let Some(len) = chrom_len { + if end > len as i64 { + end = len as i64; + } + } + if end - start < 1 { + return None; + } + + let mut newrec = record.clone(); + + let new_tlen = if tlen < 0 { + tlen - delta_tlen + } else { + tlen + delta_tlen + }; + + // Build new CIGAR: single Match for fragment span ((0, end-start)). + let span = (end - start) as u32; + let new_cigar = CigarString(vec![bam::record::Cigar::Match(span)]); + + newrec.set(record.qname(), Some(&new_cigar), &[], &[]); + + // Remove all auxiliary tags + let tags: Vec> = newrec + .aux_iter() + .filter_map(|r| r.ok().map(|(t, _)| t.to_vec())) + .collect(); + for t in &tags { + let _ = newrec.remove_aux(t); + } + + newrec.set_pos(start); + newrec.set_insert_size(new_tlen); + + // Mate position adjustment (Python: next_reference_start) + let new_mpos = { + let mut mpos = record.mpos(); + if is_read2 && is_rev { + mpos += shift[0] as i64; + } else if is_read1 && is_rev { + mpos -= shift[3] as i64; + } + mpos + }; + newrec.set_mpos(new_mpos); + + Some(newrec) +} + +fn smart_label(label: &str) -> String { + let basename = Path::new(label) + .file_name() + .and_then(|f| f.to_str()) + .unwrap_or(label); + let without_ext = basename + .split_once('.') + .map(|(name, _)| name) + .unwrap_or(basename); + if without_ext.is_empty() { + basename.to_string() + } else { + without_ext.to_string() } } diff --git a/src/filtering.rs b/src/filtering.rs index a617e4110d..3059eeae3a 100644 --- a/src/filtering.rs +++ b/src/filtering.rs @@ -1,8 +1,8 @@ -use rust_htslib::bam::{Record, ext::BamRecordExtensions, IndexedReader, Read}; -use rayon::prelude::*; -use rayon::ThreadPoolBuilder; -use crate::covcalc::Region; use crate::calc::median; +use crate::covcalc::Region; +use rayon::ThreadPoolBuilder; +use rayon::prelude::*; +use rust_htslib::bam::{IndexedReader, Read, Record, ext::BamRecordExtensions}; #[derive(Clone)] pub struct Alignmentfilters { @@ -34,7 +34,7 @@ impl Alignmentfilters { filterrnastrand: Option, extendreads: Option, extendreadslen: Option, - centerreads: Option + centerreads: Option, ) -> Self { // Go through the arguments, and if they are not set or have default values, we set a filter boolean to false. // Only when filtering needs to happen the filterrecord will be invoked, for performance. @@ -46,7 +46,7 @@ impl Alignmentfilters { let _mifl = minfraglen.unwrap_or(0); let _mafl = maxfraglen.unwrap_or(0); let _mnase = mnase.unwrap_or(false); - let _offset = offset.unwrap_or((0, 0)); + let _offset = offset.unwrap_or((0, 0)); let _frs = filterrnastrand.unwrap_or(String::from("None")); let _extend = extendreads.unwrap_or(false); let _extendreadslen = extendreadslen.unwrap_or(0); @@ -58,7 +58,14 @@ impl Alignmentfilters { } // Set the filter bool for a quick escape in case filtering is not needed. - if _mmq > 0 || _sfi > 0 || _sfe > 0 || _mifl > 0 || _mafl > 0 || _frs != "None" || blacklist.is_some() { + if _mmq > 0 + || _sfi > 0 + || _sfe > 0 + || _mifl > 0 + || _mafl > 0 + || _frs != "None" + || blacklist.is_some() + { filter = true; } @@ -71,7 +78,7 @@ impl Alignmentfilters { maxfraglen: _mafl, mnase: _mnase, offset: _offset, - filterrnastrand: _frs, + filterrnastrand: _frs, extendreads: _extend, extendreadslen: _extendreadslen, centerreads: _center, @@ -80,7 +87,7 @@ impl Alignmentfilters { } } - pub fn set_extendreadslen(&mut self, bamfile: &str, nproc: usize, regions: &Vec ) { + pub fn set_extendreadslen(&mut self, bamfile: &str, nproc: usize, regions: &Vec) { const FREAD: u16 = 0x40; let pool = ThreadPoolBuilder::new().num_threads(nproc).build().unwrap(); let fraglens: Vec = pool.install(|| { @@ -88,12 +95,15 @@ impl Alignmentfilters { .par_iter() .flat_map(|i| { let mut bam = IndexedReader::from_path(bamfile).unwrap(); - bam.fetch((i.chrom.as_str(), i.get_startu(), i.get_endu()) ) + bam.fetch((i.chrom.as_str(), i.get_startu(), i.get_endu())) .expect(&format!("Error fetching region: {:?}", i)); let mut fraglens: Vec = vec![]; for record in bam.records() { let record = record.expect("Error parsing record."); - if record.is_paired() && record.is_proper_pair() && (record.flags() & FREAD != 0) { + if record.is_paired() + && record.is_proper_pair() + && (record.flags() & FREAD != 0) + { fraglens.push(record.insert_size().abs() as u32); } } @@ -108,7 +118,7 @@ impl Alignmentfilters { panic!("No proper pairs found in the given regions. Please check your input."); } } - + pub fn filter_record(&self, rec: &Record, chrom: &str) -> bool { // Decides filtering of a record. The bool return is used to 'continue', i.e. skip the record. if rec.is_unmapped() { @@ -124,7 +134,7 @@ impl Alignmentfilters { } // samflags if self.samflaginclude > 0 { - if (rec.flags() & self.samflaginclude) == 0 { + if (rec.flags() & self.samflaginclude) != self.samflaginclude { return true; } } @@ -136,7 +146,9 @@ impl Alignmentfilters { // min/max fraglen if self.minfraglen != 0 || self.maxfraglen != 0 { if rec.is_paired() { - if rec.insert_size().abs() < self.minfraglen as i64 || rec.insert_size().abs() > self.maxfraglen as i64 { + if rec.insert_size().abs() < self.minfraglen as i64 + || rec.insert_size().abs() > self.maxfraglen as i64 + { return true; } } else { @@ -156,25 +168,28 @@ impl Alignmentfilters { if !((rec.flags() & 144 == 128) || (rec.flags() & 96 == 64)) { return true; } - }, + } ("forward", false) => { if !(rec.flags() & 16 == 16) { return true; } - }, + } ("reverse", true) => { if !((rec.flags() & 144 == 144) || (rec.flags() & 96 == 96)) { return true; } - }, + } ("reverse", false) => { if !(rec.flags() & 16 == 0) { return true; } - }, + } _ => { - panic!("filterrnastrand should be either forward or reverse. {:?} is not supported.", self.filterrnastrand) - }, + panic!( + "filterrnastrand should be either forward or reverse. {:?} is not supported.", + self.filterrnastrand + ) + } } } if self.blacklist.is_some() { @@ -189,7 +204,7 @@ impl Alignmentfilters { // In general, this is the case for MNase mode, offset, extendreads and centerreads. if self.mnase { // MNase mode, take only middle bps of the fragment - + // only retain proper pairs and forward read. let rinsertsize = rec.insert_size().abs() as u32; if rec.is_proper_pair() && !rec.is_reverse() && rinsertsize > 1 { @@ -198,24 +213,18 @@ impl Alignmentfilters { let frag_start = recpos - 1 + rinsertsize / 2; if rinsertsize % 2 == 0 { - return Some( - (frag_start..frag_start + 2).collect() - ); + return Some((frag_start..frag_start + 2).collect()); } else { - return Some( - (frag_start..frag_start+4).collect() - ); + return Some((frag_start..frag_start + 4).collect()); } } return None; } if self.offset != (0, 0) { - let mut blockvec: Vec = if self.extendreads { self.rec_extension(rec) } else { - rec - .aligned_blocks() + rec.aligned_blocks() .flat_map(|x| x[0] as u32..x[1] as u32) .collect() }; @@ -225,7 +234,11 @@ impl Alignmentfilters { // Convert potential negative indices to positive indices // It could be that for the offset only one value is given, in which case we only use that site if self.offset.1 == 0 { - let pos = if self.offset.0 < 0 {blocklen + self.offset.0 } else {self.offset.0 - 1}; + let pos = if self.offset.0 < 0 { + blocklen + self.offset.0 + } else { + self.offset.0 - 1 + }; if pos < 0 || pos >= blocklen { return None; } @@ -240,8 +253,16 @@ impl Alignmentfilters { return Some(blockvec); } } else { - let start = if self.offset.0 < 0 { blocklen + self.offset.0 } else { self.offset.0 -1}; - let end = if self.offset.1 < 0 { blocklen + self.offset.1 + 1 } else { self.offset.1 }; + let start = if self.offset.0 < 0 { + blocklen + self.offset.0 + } else { + self.offset.0 - 1 + }; + let end = if self.offset.1 < 0 { + blocklen + self.offset.1 + 1 + } else { + self.offset.1 + }; // if the range falls outside the vec, return none (retain deeptools 3 behavior) if start < 0 || end < 0 || start >= blocklen || end >= blocklen || start >= end { @@ -261,7 +282,7 @@ impl Alignmentfilters { if self.extendreads { // Extend reads let blockvec = self.rec_extension(rec); - return Some(blockvec) + return Some(blockvec); } return None; } @@ -273,14 +294,12 @@ impl Alignmentfilters { let mut blockvec: Vec = Vec::new(); let mut blocklen: u32 = 0; - rec - .aligned_blocks() - .for_each(|x| { - let _s = x[0] as u32; - let _e = x[1] as u32; - blockvec.extend(_s.._e); - blocklen += _e - _s; - }); + rec.aligned_blocks().for_each(|x| { + let _s = x[0] as u32; + let _e = x[1] as u32; + blockvec.extend(_s.._e); + blocklen += _e - _s; + }); if rec.is_proper_pair() { // Proper pairs @@ -296,7 +315,8 @@ impl Alignmentfilters { } } else { let ns = rec.reference_end() as u32; - let ne: u32 = ns + rec.insert_size().abs() as u32 - rec.seq_len_from_cigar(false) as u32; + let ne: u32 = + ns + rec.insert_size().abs() as u32 - rec.seq_len_from_cigar(false) as u32; if ns < ne { blockvec.extend(ns..ne); } @@ -323,7 +343,7 @@ impl Alignmentfilters { let ns = rec.reference_end() as u32; let ne: u32 = ns + self.extendreadslen - rec.seq_len_from_cigar(false) as u32; if ns < ne { - blockvec.extend(ns..ne ); + blockvec.extend(ns..ne); } } } @@ -334,7 +354,7 @@ impl Alignmentfilters { return blockvec; } - pub fn rec_in_blacklist(&self, rec: &Record, chrom: &str ) -> bool { + pub fn rec_in_blacklist(&self, rec: &Record, chrom: &str) -> bool { for region in self.blacklist.as_ref().unwrap().iter() { if region.chrom == chrom { let pos = rec.pos() as u32; @@ -349,5 +369,4 @@ impl Alignmentfilters { } false } - -} \ No newline at end of file +}