diff --git a/.coveragerc b/.coveragerc index 1a5a2e6..ab710af 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,2 +1,2 @@ [run] -omit = vault_service/tests* +omit = */vault_service/tests/* diff --git a/.github/workflows/python_actions.yml b/.github/workflows/python_actions.yml index 54227d6..ff86cf2 100644 --- a/.github/workflows/python_actions.yml +++ b/.github/workflows/python_actions.yml @@ -15,7 +15,7 @@ jobs: services: postgres: - image: postgres:9.6 + image: postgres:15 env: POSTGRES_DB: test POSTGRES_PASSWORD: postgres @@ -35,17 +35,15 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: '3.12' - name: Install dependencies run: | - python -m pip install --upgrade wheel setuptools==56 pip - pip install -U -r requirements.txt - pip install -U -r dev-requirements.txt + pip install ".[dev]" - name: Test with pytest run: | - py.test vault_service + pytest - name: Upload coverage data to coveralls.io run: coveralls diff --git a/README.md b/README.md index 4865ef9..54e90d7 100644 --- a/README.md +++ b/README.md @@ -14,13 +14,22 @@ Microservice for storing queries, user preferences and stuff create database vault; alter database vault owner to vault; - * create modified vault_service/local_config.py, update (at least) + * create local_config.py at the repo root (next to config.py), update (at least) VAULT_OAUTH_CLIENT_TOKEN = '.......' SQLALCHEMY_DATABASE_URI = "....." * run `alembic upgrade head` - * note: you need alembic and all dependencies in your python (`virtualenv python; pip install -r requirements; source python/bin/activate`) + * note: you need alembic and all dependencies in your python + +```bash +python3.12 -m venv python +source python/bin/activate +pip install -e ".[dev]" + +# run the tests +pytest +``` diff --git a/alembic/env.py b/alembic/env.py index bf04bbf..bfe02cd 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -1,4 +1,3 @@ -from __future__ import with_statement from alembic import context from sqlalchemy import engine_from_config, pool from logging.config import fileConfig diff --git a/alembic/versions/4cc89f2f896b_add_scix_ui_column_with_default_false.py b/alembic/versions/4cc89f2f896b_add_scix_ui_column_with_default_false.py index 93aedad..02ed684 100644 --- a/alembic/versions/4cc89f2f896b_add_scix_ui_column_with_default_false.py +++ b/alembic/versions/4cc89f2f896b_add_scix_ui_column_with_default_false.py @@ -14,7 +14,6 @@ import sqlalchemy as sa -from sqlalchemy.dialects import postgresql def upgrade(): #with app.app_context() as c: diff --git a/alembic/versions/717c2970ff42_add_get_other_papers_flag_to_myads_table.py b/alembic/versions/717c2970ff42_add_get_other_papers_flag_to_myads_table.py index 98c9e6a..b1b807f 100644 --- a/alembic/versions/717c2970ff42_add_get_other_papers_flag_to_myads_table.py +++ b/alembic/versions/717c2970ff42_add_get_other_papers_flag_to_myads_table.py @@ -14,7 +14,6 @@ import sqlalchemy as sa -from sqlalchemy.dialects import postgresql def upgrade(): #with app.app_context() as c: diff --git a/alembic/versions/af63c0205b19_adding_library_id_foreign_key_to_user.py b/alembic/versions/af63c0205b19_adding_library_id_foreign_key_to_user.py index 64de69b..b66f79a 100644 --- a/alembic/versions/af63c0205b19_adding_library_id_foreign_key_to_user.py +++ b/alembic/versions/af63c0205b19_adding_library_id_foreign_key_to_user.py @@ -14,7 +14,6 @@ import sqlalchemy as sa import json -from sqlalchemy.dialects import postgresql def upgrade(): #with app.app_context() as c: @@ -31,7 +30,7 @@ def upgrade(): # Delete link_server from user_data # But save the link_server in the library table instead to be accessible via users.library_id - users = bind.execute("SELECT id, user_data FROM users WHERE user_data ? 'link_server'") + users = bind.execute(sa.text("SELECT id, user_data FROM users WHERE user_data ? 'link_server'")) for user_id, user_data in users: if user_data and isinstance(user_data, dict): @@ -40,7 +39,7 @@ def upgrade(): library_id = None if link_server: # Only look up library if link_server is not empty library_result = bind.execute( - "SELECT id FROM library WHERE libserver = %(libserver)s", + sa.text("SELECT id FROM library WHERE libserver = :libserver"), {"libserver": link_server} ).fetchone() library_id = library_result[0] if library_result else None @@ -50,7 +49,7 @@ def upgrade(): # Convert dictionary to JSON string for PostgreSQL bind.execute( - "UPDATE users SET user_data = %(user_data)s, library_id = %(library_id)s WHERE id = %(user_id)s", + sa.text("UPDATE users SET user_data = :user_data, library_id = :library_id WHERE id = :user_id"), { "user_data": json.dumps(new_user_data), "library_id": library_id, @@ -63,7 +62,7 @@ def downgrade(): bind = op.get_bind() - result = bind.execute("SELECT u.id, u.user_data, l.libserver FROM users u JOIN library l ON u.library_id = l.id WHERE u.library_id IS NOT NULL") + result = bind.execute(sa.text("SELECT u.id, u.user_data, l.libserver FROM users u JOIN library l ON u.library_id = l.id WHERE u.library_id IS NOT NULL")) for user_id, user_data, libserver in result: user_data = user_data or {} @@ -71,7 +70,7 @@ def downgrade(): if libserver is not None: user_data['link_server'] = libserver bind.execute( - "UPDATE users SET user_data = %(user_data)s WHERE id = %(user_id)s", + sa.text("UPDATE users SET user_data = :user_data WHERE id = :user_id"), { "user_data": json.dumps(user_data), "user_id": user_id @@ -80,12 +79,12 @@ def downgrade(): # Find and drop the actual foreign key constraint bind = op.get_bind() - result = bind.execute(""" + result = bind.execute(sa.text(""" SELECT constraint_name FROM information_schema.table_constraints WHERE table_name = 'users' AND constraint_type = 'FOREIGN KEY' - """).fetchone() + """)).fetchone() if result: constraint_name = result[0] diff --git a/alembic/versions/ffdbd392dc89_myads_nullable_fix.py b/alembic/versions/ffdbd392dc89_myads_nullable_fix.py index 7e31062..9cba47b 100644 --- a/alembic/versions/ffdbd392dc89_myads_nullable_fix.py +++ b/alembic/versions/ffdbd392dc89_myads_nullable_fix.py @@ -11,7 +11,6 @@ down_revision = 'c342556e5569' from alembic import op -import sqlalchemy as sa diff --git a/dev-requirements.txt b/dev-requirements.txt deleted file mode 100644 index 107b644..0000000 --- a/dev-requirements.txt +++ /dev/null @@ -1,11 +0,0 @@ -flask-cors==2.0.0 -Flask-Testing==0.8.1 -httpretty==0.9.5 -coveralls==2.2.0 -coverage==5.2.1 -pep257==0.5.0 -pytest==6.2.1 -pytest-cache==1.0 -pytest-cov==2.10.1 -pytest-pep8==1.0.6 -testing.postgresql==1.3.0 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..e1e85c0 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,50 @@ +[project] +name = "vault_service" +version = "0.0.1" +description = "ADS Vault Service" +authors = [{ name = "ADS Team", email = "adshelp@cfa.harvard.edu" }] +license = { text = "MIT" } +readme = "README.md" +requires-python = ">=3.12" +dependencies = [ + "adsmutils @ git+https://github.com/adsabs/ADSMicroserviceUtils.git@v2.0.0", + "ADSParser @ git+https://github.com/adsabs/ADSParser.git@v1.0.4", + "Flask-Cors==6.0.5", + "alembic==1.19.1", + "psycopg2-binary==2.9.12", + "Jinja2==3.1.6", + "markupsafe==3.0.3", + "itsdangerous==2.2.0", + "werkzeug==2.3.8" +] + +[project.optional-dependencies] +dev = [ + "Flask-Testing==0.8.1", + "httpretty==1.1.4", + "testing.postgresql==1.3.0", + "pytest==9.1.1", + "pytest-cov==7.1.0", + "coverage==7.16.0", + "coveralls==4.1.0" +] + +[build-system] +requires = ["setuptools>=62.0.0", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +include = ["vault_service*"] +exclude = ["vault_service.tests*"] + +[tool.pytest.ini_options] +addopts = "--cov=vault_service --cov-report=term-missing" +testpaths = ["vault_service/tests"] +pythonpath = ["."] + +[tool.black] +line-length = 88 +target-version = ['py312'] + +[tool.isort] +profile = "black" diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index fb4eab4..0000000 --- a/pytest.ini +++ /dev/null @@ -1,3 +0,0 @@ -[pytest] -addopts = --cov=vault_service --cov-report=term-missing -testpaths = vault_service/tests diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 1fb87a3..0000000 --- a/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -git+https://github.com/adsabs/ADSMicroserviceUtils.git@v1.2.3 -git+https://github.com/adsabs/ADSParser.git@v1.0.4 -alembic==0.8.9 -psycopg2==2.8.6 -Flask-Script==2.0.5 -itsdangerous<=2.0.1 -werkzeug<=2.0.3 -markupsafe<=2.0.1 -jinja2==2.11.3 diff --git a/vault_service/app.py b/vault_service/app.py index b4084fd..526e6aa 100644 --- a/vault_service/app.py +++ b/vault_service/app.py @@ -1,4 +1,3 @@ -import logging.config from werkzeug.serving import run_simple import os, sys, inspect @@ -26,7 +25,7 @@ def create_app(**config): ## http://docs.sqlalchemy.org/en/rel_0_9/dialects/sqlite.html#pysqlite-serializable if 'sqlite' in app.config.get('SQLALCHEMY_DATABASE_URI', None): - from sqlalchemy import event + from sqlalchemy import event, text engine = app.db.engine @event.listens_for(engine, "connect") @@ -38,7 +37,7 @@ def do_connect(dbapi_connection, connection_record): @event.listens_for(engine, "begin") def do_begin(conn): # emit our own BEGIN - conn.execute("BEGIN") + conn.execute(text("BEGIN")) # Note about imports being here rather than at the top level diff --git a/vault_service/models.py b/vault_service/models.py index b3abd32..67760ea 100644 --- a/vault_service/models.py +++ b/vault_service/models.py @@ -7,7 +7,7 @@ """ from sqlalchemy import Column, Integer, String, LargeBinary, ForeignKey, Boolean, Text from sqlalchemy.dialects.postgresql import JSONB, ENUM, ARRAY -from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import declarative_base from sqlalchemy.ext.mutable import MutableDict from adsmutils import UTCDateTime, get_date import sqlalchemy as sa diff --git a/vault_service/tests/test_bumblebee.py b/vault_service/tests/test_bumblebee.py index fc048ea..fff61da 100644 --- a/vault_service/tests/test_bumblebee.py +++ b/vault_service/tests/test_bumblebee.py @@ -1,15 +1,13 @@ import sys, os -from urllib.parse import urlencode -from flask import url_for, request +from flask import url_for import unittest -import json project_home = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../')) if project_home not in sys.path: sys.path.insert(0, project_home) from vault_service import app -from vault_service.models import Query, Institute, Library, Base +from vault_service.models import Institute, Library, Base from vault_service.tests.base import TestCaseDatabase class TestSite(TestCaseDatabase): diff --git a/vault_service/tests/test_query_as_monument.py b/vault_service/tests/test_query_as_monument.py index 54d2081..ca771a3 100644 --- a/vault_service/tests/test_query_as_monument.py +++ b/vault_service/tests/test_query_as_monument.py @@ -1,11 +1,7 @@ import sys, os -from urllib.parse import urlencode -from flask import url_for, request +from flask import url_for import unittest import json -import httpretty -import cgi -from io import StringIO project_home = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../')) if project_home not in sys.path: diff --git a/vault_service/tests/test_service.py b/vault_service/tests/test_service.py index 8cd9587..ce31ecd 100644 --- a/vault_service/tests/test_service.py +++ b/vault_service/tests/test_service.py @@ -1,16 +1,10 @@ import sys, os -from urllib.parse import urlencode project_home = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../')) if project_home not in sys.path: sys.path.insert(0, project_home) -from flask import url_for, request import unittest -import json -import httpretty -import cgi -from io import StringIO from vault_service.tests.base import TestCaseDatabase class TestServices(TestCaseDatabase): diff --git a/vault_service/tests/test_utils.py b/vault_service/tests/test_utils.py index 7b63353..9142f0f 100644 --- a/vault_service/tests/test_utils.py +++ b/vault_service/tests/test_utils.py @@ -1,20 +1,16 @@ import sys, os -from urllib.parse import urlencode -from flask import url_for, request +from flask import url_for import unittest import json import httpretty -import cgi -from io import StringIO project_home = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../')) if project_home not in sys.path: sys.path.insert(0, project_home) -from vault_service.models import Query, User, MyADS +from vault_service.models import MyADS from vault_service.views import utils from vault_service.tests.base import TestCaseDatabase -import adsmutils class TestServices(TestCaseDatabase): diff --git a/vault_service/views/user.py b/vault_service/views/user.py index 2e741f6..cf06be3 100644 --- a/vault_service/views/user.py +++ b/vault_service/views/user.py @@ -1,6 +1,6 @@ from flask import Blueprint from flask import current_app -from flask import request, url_for +from flask import request import json from hashlib import md5 diff --git a/vault_service/views/utils.py b/vault_service/views/utils.py index c545da0..8ee8955 100644 --- a/vault_service/views/utils.py +++ b/vault_service/views/utils.py @@ -9,7 +9,6 @@ from sqlalchemy import exc from sqlalchemy.orm import exc as ormexc -from sqlalchemy.sql.expression import all_ def make_solr_request(query, bigquery=None, headers=None):