Skip to content

Commit 8a00969

Browse files
mahmoudmahmoud
committed
mock out stuff to get CI unblocked for now, will add vcrpy soonish
1 parent a67db43 commit 8a00969

6 files changed

Lines changed: 269 additions & 12 deletions

File tree

‎.github/workflows/test-backend.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ jobs:
2121
-v "${{ github.workspace }}":/app \
2222
-e PYTHONPATH=/app \
2323
montage-ci \
24-
python -m pytest montage/tests/test_web_basic.py \
24+
python -m pytest montage/tests/ \
2525
-v --tb=short -p no:cacheprovider
2626
2727
- name: Verify backend imports

‎dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,6 @@ COPY requirements.txt .
88

99
RUN pip install --upgrade pip
1010
RUN pip install -r requirements.txt
11-
RUN pip install pytest
11+
RUN pip install pytest responses
1212

1313
EXPOSE 5000

‎montage/tests/conftest.py‎

Lines changed: 193 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,193 @@
1+
# -*- coding: utf-8 -*-
2+
"""
3+
Shared test fixtures.
4+
5+
Mocks all external HTTP calls (Toolforge API, Google Sheets, Wikimedia
6+
API) so tests run without network access. The ``responses`` library
7+
intercepts at the ``requests`` adapter level; any unmocked call raises
8+
``ConnectionError`` (passthrough=False, the default).
9+
"""
10+
11+
from __future__ import absolute_import
12+
13+
import json
14+
import re
15+
16+
import pytest
17+
import responses as responses_lib
18+
19+
from urllib.parse import parse_qs, urlparse
20+
21+
# ---------------------------------------------------------------------------
22+
# URLs exactly as constructed by montage code
23+
# ---------------------------------------------------------------------------
24+
TOOLFORGE_CATEGORY_URL = 'https://montage.toolforge.org/v1/utils//category'
25+
TOOLFORGE_FILE_URL = 'https://montage.toolforge.org/v1/utils//file'
26+
27+
# Matches any Google Sheets CSV-export URL regardless of doc ID.
28+
GSHEET_CSV_URL_RE = re.compile(
29+
r'https://docs\.google\.com/spreadsheets/d/.+/gviz/tq\?tqx=out:csv'
30+
)
31+
32+
# Matches any Wikimedia API user-lookup call.
33+
MW_API_URL_RE = re.compile(
34+
r'https://commons\.wikimedia\.org/w/api\.php\?.*'
35+
)
36+
37+
# ---------------------------------------------------------------------------
38+
# Fixture data -- 20 synthetic entries with resolution > 2 megapixels.
39+
# Enough entries to survive disqualification AND give every juror >=2
40+
# tasks regardless of random.shuffle ordering during task allocation.
41+
# ---------------------------------------------------------------------------
42+
43+
44+
def _generate_file_infos(n):
45+
"""Build *n* unique file-info dicts with high resolution."""
46+
infos = []
47+
for i in range(n):
48+
infos.append({
49+
'img_name': 'Test_WLM_2015_image_%03d.jpg' % (i + 1),
50+
'img_major_mime': 'image',
51+
'img_minor_mime': 'jpeg',
52+
'img_width': '3264',
53+
'img_height': '2448', # 3264*2448 = 7,990,272 > 2M
54+
'img_user': '5193613',
55+
'img_user_text': 'Khoshamadgou',
56+
# All timestamps after campaign open_date (2015-09-01)
57+
'img_timestamp': '201509060%05d' % (20000 + i),
58+
})
59+
return infos
60+
61+
62+
FIXTURE_FILE_INFOS = _generate_file_infos(20)
63+
64+
# Entry returned for the single-filename import in test_web_basic.py
65+
SELECTED_FILE_INFO = {
66+
'img_name': u'Reynisfjara, Su\u00f0urland, Islandia, 2014-08-17, DD 164.JPG',
67+
'img_major_mime': 'image',
68+
'img_minor_mime': 'jpeg',
69+
'img_width': '4928',
70+
'img_height': '3280',
71+
'img_user': '12345',
72+
'img_user_text': 'TestUploader',
73+
'img_timestamp': '20140817120000',
74+
}
75+
76+
CSV_FULL_COLS = [
77+
'img_name', 'img_major_mime', 'img_minor_mime',
78+
'img_width', 'img_height', 'img_user',
79+
'img_user_text', 'img_timestamp',
80+
]
81+
82+
83+
def build_full_csv(file_infos=None):
84+
"""Build a CSV string with all required columns from file_info dicts."""
85+
if file_infos is None:
86+
file_infos = FIXTURE_FILE_INFOS
87+
lines = [','.join(CSV_FULL_COLS)]
88+
for fi in file_infos:
89+
lines.append(','.join(str(fi[c]) for c in CSV_FULL_COLS))
90+
return '\n'.join(lines) + '\n'
91+
92+
93+
def build_filename_csv(file_infos=None):
94+
"""Build a CSV string with only a 'filename' column."""
95+
if file_infos is None:
96+
file_infos = FIXTURE_FILE_INFOS
97+
lines = ['filename']
98+
for fi in file_infos:
99+
lines.append(fi['img_name'])
100+
return '\n'.join(lines) + '\n'
101+
102+
103+
FIXTURE_FULL_CSV = build_full_csv()
104+
FIXTURE_FILENAME_CSV = build_filename_csv()
105+
106+
107+
# ---------------------------------------------------------------------------
108+
# Disable pdb in error handler -- devtest sets debug_errors=True which
109+
# calls pdb.post_mortem() on unhandled exceptions. Under pytest's output
110+
# capture this crashes with OSError. Patching pdb to no-ops is safe
111+
# because no test relies on interactive debugging.
112+
# ---------------------------------------------------------------------------
113+
@pytest.fixture(autouse=True)
114+
def _disable_pdb(monkeypatch):
115+
monkeypatch.setattr('pdb.set_trace', lambda *a, **kw: None)
116+
monkeypatch.setattr('pdb.post_mortem', lambda *a, **kw: None)
117+
118+
# ---------------------------------------------------------------------------
119+
# Wikimedia API callback -- returns a plausible user record for any username
120+
# ---------------------------------------------------------------------------
121+
def _wikimedia_user_callback(request):
122+
"""Return a mock globalallusers response matching the requested username."""
123+
parsed = urlparse(request.url)
124+
params = parse_qs(parsed.query)
125+
username = params.get('agufrom', ['Unknown'])[0]
126+
# Deterministic fake user ID derived from username
127+
user_id = abs(hash(username)) % 10**8
128+
body = json.dumps({
129+
'query': {
130+
'globalallusers': [
131+
{'name': username, 'id': str(user_id)}
132+
]
133+
}
134+
})
135+
return (200, {}, body)
136+
137+
138+
# ---------------------------------------------------------------------------
139+
# Fixture: mock_external_apis
140+
# ---------------------------------------------------------------------------
141+
@pytest.fixture
142+
def mock_external_apis():
143+
"""Activate ``responses`` and register mocks for every external endpoint.
144+
145+
Covers:
146+
- Toolforge category lookup (POST /v1/utils//category)
147+
- Toolforge file lookup (POST /v1/utils//file)
148+
- Google Sheets CSV export (GET docs.google.com/spreadsheets/...)
149+
- Wikimedia user lookup (GET commons.wikimedia.org/w/api.php)
150+
151+
Any request to an unregistered URL raises ``ConnectionError``,
152+
ensuring no live HTTP traffic leaks from tests.
153+
"""
154+
with responses_lib.RequestsMock(assert_all_requests_are_fired=False) as rsps:
155+
# -- Toolforge category endpoint --
156+
rsps.add(
157+
responses_lib.POST,
158+
TOOLFORGE_CATEGORY_URL,
159+
json={'file_infos': FIXTURE_FILE_INFOS, 'no_info': []},
160+
status=200,
161+
)
162+
163+
# -- Toolforge file-lookup endpoint --
164+
# Returns both fixture entries and the single "selected" entry so
165+
# that both bulk-filename and single-filename imports succeed.
166+
rsps.add(
167+
responses_lib.POST,
168+
TOOLFORGE_FILE_URL,
169+
json={
170+
'file_infos': FIXTURE_FILE_INFOS + [SELECTED_FILE_INFO],
171+
'no_info': [],
172+
},
173+
status=200,
174+
)
175+
176+
# -- Google Sheets CSV export (any doc ID) --
177+
rsps.add(
178+
responses_lib.GET,
179+
GSHEET_CSV_URL_RE,
180+
body=FIXTURE_FULL_CSV,
181+
status=200,
182+
content_type='text/csv',
183+
)
184+
185+
# -- Wikimedia user-lookup API --
186+
# Called by get_mw_userid() when creating new users.
187+
rsps.add_callback(
188+
responses_lib.GET,
189+
MW_API_URL_RE,
190+
callback=_wikimedia_user_callback,
191+
)
192+
193+
yield rsps

‎montage/tests/test_loaders.py‎

Lines changed: 70 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,92 @@
11
# -*- coding: utf-8 -*-
22

33
from __future__ import print_function
4-
54
from __future__ import absolute_import
5+
6+
import responses
67
from pytest import raises
78

89
from montage.loaders import get_entries_from_gsheet
910

11+
from .conftest import (
12+
FIXTURE_FILE_INFOS,
13+
FIXTURE_FULL_CSV,
14+
FIXTURE_FILENAME_CSV,
15+
TOOLFORGE_FILE_URL,
16+
)
17+
1018
RESULTS = 'https://docs.google.com/spreadsheets/d/1RDlpT23SV_JB1mIz0OA-iuc3MNdNVLbaK_LtWAC7vzg/edit?usp=sharing'
1119
FILENAME_LIST = 'https://docs.google.com/spreadsheets/d/1Nqj-JsX3L5qLp5ITTAcAFYouglbs5OpnFwP6zSFpa0M/edit?usp=sharing'
1220
GENERIC_CSV = 'https://docs.google.com/spreadsheets/d/1WzHFg_bhvNthRMwNmxnk010KJ8fwuyCrby29MvHUzH8/edit#gid=550467819'
1321
FORBIDDEN_SHEET = 'https://docs.google.com/spreadsheets/d/1tza92brMKkZBTykw3iS6X9ij1D4_kvIYAiUlq1Yi7Fs/edit'
1422

23+
# Pre-compute the actual fetch URLs that loaders.py constructs from the
24+
# doc IDs embedded in the spreadsheet URLs above.
25+
_RESULTS_CSV_URL = 'https://docs.google.com/spreadsheets/d/1RDlpT23SV_JB1mIz0OA-iuc3MNdNVLbaK_LtWAC7vzg/gviz/tq?tqx=out:csv'
26+
_FILENAME_CSV_URL = 'https://docs.google.com/spreadsheets/d/1Nqj-JsX3L5qLp5ITTAcAFYouglbs5OpnFwP6zSFpa0M/gviz/tq?tqx=out:csv'
27+
_GENERIC_CSV_URL = 'https://docs.google.com/spreadsheets/d/1WzHFg_bhvNthRMwNmxnk010KJ8fwuyCrby29MvHUzH8/gviz/tq?tqx=out:csv'
28+
_FORBIDDEN_CSV_URL = 'https://docs.google.com/spreadsheets/d/1tza92brMKkZBTykw3iS6X9ij1D4_kvIYAiUlq1Yi7Fs/gviz/tq?tqx=out:csv'
29+
30+
31+
@responses.activate
1532
def test_load_results():
33+
"""Full CSV with all required columns -- entries created directly."""
34+
responses.add(
35+
responses.GET,
36+
_RESULTS_CSV_URL,
37+
body=FIXTURE_FULL_CSV,
38+
status=200,
39+
content_type='text/csv',
40+
)
1641
imgs, warnings = get_entries_from_gsheet(RESULTS, source='remote')
17-
assert len(imgs) == 331
42+
assert len(imgs) == len(FIXTURE_FILE_INFOS)
43+
1844

45+
@responses.activate
1946
def test_load_filenames():
20-
imgs, warnings = get_entries_from_gsheet(FILENAME_LIST, source='remote')
21-
assert len(imgs) == 89
47+
"""Partial CSV with only 'filename' column -- triggers Toolforge lookup."""
48+
responses.add(
49+
responses.GET,
50+
_FILENAME_CSV_URL,
51+
body=FIXTURE_FILENAME_CSV,
52+
status=200,
53+
content_type='text/csv',
54+
)
55+
# The filename-only CSV triggers load_partial_csv -> load_name_list
56+
# -> get_by_filename_remote -> POST to Toolforge /file endpoint.
57+
responses.add(
58+
responses.POST,
59+
TOOLFORGE_FILE_URL,
60+
json={'file_infos': FIXTURE_FILE_INFOS, 'no_info': []},
61+
status=200,
62+
)
63+
imgs, warnings = get_entries_from_gsheet(FILENAME_LIST, source='remote')
64+
assert len(imgs) == len(FIXTURE_FILE_INFOS)
2265

66+
67+
@responses.activate
2368
def test_load_csv():
24-
imgs, warnings = get_entries_from_gsheet(GENERIC_CSV, source='remote')
25-
assert len(imgs) == 93
69+
"""Generic full CSV -- same path as test_load_results."""
70+
responses.add(
71+
responses.GET,
72+
_GENERIC_CSV_URL,
73+
body=FIXTURE_FULL_CSV,
74+
status=200,
75+
content_type='text/csv',
76+
)
77+
imgs, warnings = get_entries_from_gsheet(GENERIC_CSV, source='remote')
78+
assert len(imgs) == len(FIXTURE_FILE_INFOS)
79+
2680

81+
@responses.activate
2782
def test_no_persmission():
83+
"""Non-CSV content-type signals a permission / sharing error."""
84+
responses.add(
85+
responses.GET,
86+
_FORBIDDEN_CSV_URL,
87+
body='<html><body>Sign in</body></html>',
88+
status=200,
89+
content_type='text/html',
90+
)
2891
with raises(ValueError):
29-
imgs, warnings = get_entries_from_gsheet(FORBIDDEN_SHEET, source='remote')
92+
get_entries_from_gsheet(FORBIDDEN_SHEET, source='remote')

‎montage/tests/test_web_basic.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,7 @@ def api_client(montage_app):
165165
return api_client
166166

167167

168-
def test_home_client(base_client, api_client):
168+
def test_home_client(base_client, api_client, mock_external_apis):
169169

170170
resp = base_client.fetch('organizer: home', '/')
171171
#resp = base_client.fetch('public: login', '/login')
@@ -821,7 +821,7 @@ def test_home_client(base_client, api_client):
821821
#resp = base_client.fetch('public: logout', '/logout')
822822

823823

824-
def test_multiple_jurors(api_client):
824+
def test_multiple_jurors(api_client, mock_external_apis):
825825
# This is copied from above. What's the best way to break up the tests into
826826
# various stages? Should I use a pytest.fixture?
827827
fetch = api_client.fetch

‎requirements-dev.txt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,5 @@
33
tox<3.15.0
44
Fabric3
55
coverage==5.0.2
6-
pytest==4.6.9
6+
pytest>=7,<9
7+
responses>=0.25.0

0 commit comments

Comments
 (0)