diff --git a/src/grimoirelab/core/config/settings.py b/src/grimoirelab/core/config/settings.py index 4f12a56..806e286 100644 --- a/src/grimoirelab/core/config/settings.py +++ b/src/grimoirelab/core/config/settings.py @@ -419,3 +419,10 @@ MATCH_TRUSTED_SOURCES = os.environ.get( "SORTINGHAT_MATCH_TRUSTED_SOURCES", "github,gitlab,slack" ).split(",") + + +# +# TODO: Obtain the tokens from a secure vault +# GitLab tokens +# +GRIMOIRELAB_GITLAB_TOKENS = os.environ.get("GRIMOIRELAB_GITLAB_TOKENS", "").split(",") diff --git a/src/grimoirelab/core/consumers/identities.py b/src/grimoirelab/core/consumers/identities.py index 56392c5..a8da717 100644 --- a/src/grimoirelab/core/consumers/identities.py +++ b/src/grimoirelab/core/consumers/identities.py @@ -45,6 +45,12 @@ GIT_EVENT_COMMIT_TESTED_BY, ) +from chronicler.events.core.gitlab import ( + GITLAB_EVENT_AUTHOR, + GITLAB_EVENT_ASSIGNEE, + GITLAB_EVENT_CLOSED_BY, +) + IDENTITY_EVENTS = ( GIT_EVENT_COMMIT_AUTHORED_BY, @@ -58,6 +64,9 @@ GIT_EVENT_COMMIT_SIGNED_OFF_BY, GIT_EVENT_COMMIT_SUGGESTED_BY, GIT_EVENT_COMMIT_TESTED_BY, + GITLAB_EVENT_AUTHOR, + GITLAB_EVENT_ASSIGNEE, + GITLAB_EVENT_CLOSED_BY, ) diff --git a/src/grimoirelab/core/scheduler/tasks/chronicler.py b/src/grimoirelab/core/scheduler/tasks/chronicler.py index 4fd775d..322e456 100644 --- a/src/grimoirelab/core/scheduler/tasks/chronicler.py +++ b/src/grimoirelab/core/scheduler/tasks/chronicler.py @@ -28,6 +28,8 @@ import perceval.backends import chronicler.eventizer +from urllib.parse import urlparse +from django.conf import settings from grimoirelab_toolkit.datetime import str_to_datetime from ...scheduler.errors import NotFoundError @@ -77,6 +79,12 @@ def chronicler_job( backend_args = job_args.copy() if job_args else {} + # Convert common datetime arguments + datetime_args = ["from_date", "to_date"] + for arg in datetime_args: + if arg in backend_args and isinstance(backend_args[arg], str): + backend_args[arg] = str_to_datetime(backend_args[arg]) + # Get the generator to fetch the data items perceval_gen = perceval.backend.BackendItemsGenerator( backend_class, backend_args, datasource_category @@ -222,7 +230,7 @@ def resuming_args( params = {} if progress.summary and progress.summary.fetched > 0: - params["from_date"] = progress.summary.max_updated_on + params["from_date"] = progress.summary.max_updated_on.isoformat() if progress.summary.max_offset: params["offset"] = progress.summary.max_offset @@ -239,12 +247,13 @@ def recovery_args( return ChroniclerArgumentGenerator.resuming_args(task_args, progress) -def get_chronicler_argument_generator(name: str) -> ChroniclerArgumentGenerator: +def get_chronicler_argument_generator(name: str) -> type[ChroniclerArgumentGenerator]: """Get the argument generator for a backend.""" generators = { "git": GitArgumentGenerator, "github": GitHubArgumentGenerator, + "gitlab": GitLabArgumentGenerator, } return generators.get(name.lower(), ChroniclerArgumentGenerator) @@ -359,7 +368,76 @@ def recovery_args( return job_args -class GitLabArgumentGenerator(GitHubArgumentGenerator): +class GitLabArgumentGenerator(ChroniclerArgumentGenerator): """Chronicler argument generator for GitLab.""" - pass + @staticmethod + def initial_args(task_args: dict[str, Any]) -> dict[str, Any]: + """GitLab initial arguments.""" + + # For the first execution make some arguments mandatory + job_args = {} + + domain, owner, repo = GitLabArgumentGenerator.parse_url(task_args["uri"]) + + job_args["owner"] = owner + job_args["repository"] = repo + if domain != "gitlab.com": + job_args["enterprise_url"] = f"https://{domain}" + + # TODO: Obtain tokens from a secure storage + tokens = task_args.get("api_token", settings.GRIMOIRELAB_GITLAB_TOKENS)[0] + + job_args["api_token"] = tokens + job_args["sleep_for_rate"] = True + + return job_args + + @staticmethod + def parse_url(url: str) -> tuple[str, str, str]: + """Parse GitLab URL to Perceval format. + + If a given GitLab repository is under more than 1 level, + all the slashes starting from the second level have to be + replaced by `%2F`. + + :param url: GitLab repository URL + :return: A tuple with (domain, owner, repository) + """ + parsed = urlparse(url) + parts = [p for p in parsed.path.split("/") if p] + if len(parts) < 2: + raise ValueError(f"Invalid GitLab URL: {url}") + + domain = parsed.netloc + owner = parts[0] + repo = "%2F".join(parts[1:]).removesuffix(".git") + + return domain, owner, repo + + @staticmethod + def resuming_args( + task_args: dict[str, Any] | None, + progress: ChroniclerProgress, + ) -> dict[str, Any]: + """GitHub resuming arguments.""" + + job_args = task_args.copy() if task_args else {} + job_args["sleep_for_rate"] = True + job_args["from_date"] = progress.summary.last_updated_on.isoformat() + + return job_args + + @staticmethod + def recovery_args( + task_args: dict[str, Any], + progress: ChroniclerProgress, + ) -> dict[str, Any]: + """GitHub recovery arguments.""" + + job_args = task_args.copy() if task_args else {} + + if progress.summary and progress.summary.last_updated_on: + job_args["from_date"] = progress.summary.last_updated_on.isoformat() + + return job_args diff --git a/tests/unit/scheduler/data/gitlab_issues.json b/tests/unit/scheduler/data/gitlab_issues.json new file mode 100644 index 0000000..df7b99c --- /dev/null +++ b/tests/unit/scheduler/data/gitlab_issues.json @@ -0,0 +1,488 @@ +[ + { + "backend_name":"GitLab", + "backend_version":"1.0.0", + "category":"issue", + "classified_fields_filtered":null, + "data":{ + "_links":{ + "award_emoji":"https://gitlab.com/api/v4/projects/12345678/issues/2/award_emoji", + "closed_as_duplicate_of":null, + "notes":"https://gitlab.com/api/v4/projects/12345678/issues/2/notes", + "project":"https://gitlab.com/api/v4/projects/12345678", + "self":"https://gitlab.com/api/v4/projects/12345678/issues/2" + }, + "assignee":null, + "assignees":[ ], + "author":{ + "avatar_url":"https://secure.gravatar.com/avatar/xxxxxxxx?s=80&d=identicon", + "id":8276902, + "locked":false, + "name":"User Test", + "public_email":"", + "state":"active", + "username":"user_1", + "web_url":"https://gitlab.com/user_1" + }, + "award_emoji_data":[ ], + "blocking_issues_count":0, + "closed_at":"2026-01-20T09:56:45.944Z", + "closed_by":{ + "avatar_url":"https://secure.gravatar.com/avatar/xxxxxxxx?s=80&d=identicon", + "id":8276902, + "locked":false, + "name":"User Test", + "public_email":"", + "state":"active", + "username":"user_1", + "web_url":"https://gitlab.com/user_1" + }, + "confidential":false, + "created_at":"2026-01-20T09:56:26.236Z", + "description":"Body issue 2", + "discussion_locked":null, + "downvotes":0, + "due_date":null, + "has_tasks":true, + "id":181476659, + "iid":2, + "imported":false, + "imported_from":"none", + "issue_type":"issue", + "labels":[ ], + "merge_requests_count":0, + "milestone":null, + "moved_to_id":null, + "notes_data":[ ], + "project_id":12345678, + "references":{ + "full":"user_1/tmp#2", + "relative":"#2", + "short":"#2" + }, + "service_desk_reply_to":null, + "severity":"UNKNOWN", + "state":"closed", + "task_completion_status":{ + "completed_count":0, + "count":0 + }, + "task_status":"0 of 0 checklist items completed", + "time_stats":{ + "human_time_estimate":null, + "human_total_time_spent":null, + "time_estimate":0, + "total_time_spent":0 + }, + "title":"Example issue 2", + "type":"ISSUE", + "updated_at":"2026-01-20T09:56:46.324Z", + "upvotes":0, + "user_notes_count":0, + "web_url":"https://gitlab.com/user_1/tmp/-/issues/2" + }, + "origin":"https://gitlab.com/user_1/tmp", + "perceval_version":"1.4.4", + "search_fields":{ + "groups":null, + "iid":2, + "item_id":"181476659", + "owner":"user_1", + "project":"tmp" + }, + "tag":"https://gitlab.com/user_1/tmp", + "timestamp":1.768904305608975E9, + "updated_on":1.768903006324E9, + "uuid":"cbd19489e7812e32f5f26473e202af1a892c0a65" + }, + { + "backend_name":"GitLab", + "backend_version":"1.0.0", + "category":"issue", + "classified_fields_filtered":null, + "data":{ + "_links":{ + "award_emoji":"https://gitlab.com/api/v4/projects/12345678/issues/3/award_emoji", + "closed_as_duplicate_of":null, + "notes":"https://gitlab.com/api/v4/projects/12345678/issues/3/notes", + "project":"https://gitlab.com/api/v4/projects/12345678", + "self":"https://gitlab.com/api/v4/projects/12345678/issues/3" + }, + "assignee":null, + "assignees":[ ], + "author":{ + "avatar_url":"https://secure.gravatar.com/avatar/xxxxxxxx?s=80&d=identicon", + "id":8276902, + "locked":false, + "name":"User Test", + "public_email":"", + "state":"active", + "username":"user_1", + "web_url":"https://gitlab.com/user_1" + }, + "award_emoji_data":[ ], + "blocking_issues_count":0, + "closed_at":null, + "closed_by":null, + "confidential":false, + "created_at":"2026-01-20T09:57:03.162Z", + "description":"Body incident", + "discussion_locked":null, + "downvotes":0, + "due_date":null, + "has_tasks":true, + "id":181476677, + "iid":3, + "imported":false, + "imported_from":"none", + "issue_type":"incident", + "labels":[ ], + "merge_requests_count":0, + "milestone":null, + "moved_to_id":null, + "notes_data":[ ], + "project_id":12345678, + "references":{ + "full":"user_1/tmp#3", + "relative":"#3", + "short":"#3" + }, + "service_desk_reply_to":null, + "severity":"UNKNOWN", + "state":"opened", + "task_completion_status":{ + "completed_count":0, + "count":0 + }, + "task_status":"0 of 0 checklist items completed", + "time_stats":{ + "human_time_estimate":null, + "human_total_time_spent":null, + "time_estimate":0, + "total_time_spent":0 + }, + "title":"Example incident", + "type":"INCIDENT", + "updated_at":"2026-01-20T09:57:03.162Z", + "upvotes":0, + "user_notes_count":0, + "web_url":"https://gitlab.com/user_1/tmp/-/issues/3" + }, + "origin":"https://gitlab.com/user_1/tmp", + "perceval_version":"1.4.4", + "search_fields":{ + "groups":null, + "iid":3, + "item_id":"181476677", + "owner":"user_1", + "project":"tmp" + }, + "tag":"https://gitlab.com/user_1/tmp", + "timestamp":1.768904306188272E9, + "updated_on":1.768903023162E9, + "uuid":"8ea9a635f395f8dadafd932c59b1e42e754eefd0" + }, + { + "backend_name":"GitLab", + "backend_version":"1.0.0", + "category":"issue", + "classified_fields_filtered":null, + "data":{ + "_links":{ + "award_emoji":"https://gitlab.com/api/v4/projects/12345678/issues/4/award_emoji", + "closed_as_duplicate_of":null, + "notes":"https://gitlab.com/api/v4/projects/12345678/issues/4/notes", + "project":"https://gitlab.com/api/v4/projects/12345678", + "self":"https://gitlab.com/api/v4/projects/12345678/issues/4" + }, + "assignee":{ + "avatar_url":"https://secure.gravatar.com/avatar/xxxxxxxx?s=80&d=identicon", + "id":8276902, + "locked":false, + "name":"User Test", + "public_email":"", + "state":"active", + "username":"user_1", + "web_url":"https://gitlab.com/user_1" + }, + "assignees":[ + { + "avatar_url":"https://secure.gravatar.com/avatar/xxxxxxxx?s=80&d=identicon", + "id":8276902, + "locked":false, + "name":"User Test", + "public_email":"", + "state":"active", + "username":"user_1", + "web_url":"https://gitlab.com/user_1" + } + ], + "author":{ + "avatar_url":"https://secure.gravatar.com/avatar/xxxxxxxx?s=80&d=identicon", + "id":8276902, + "locked":false, + "name":"User Test", + "public_email":"", + "state":"active", + "username":"user_1", + "web_url":"https://gitlab.com/user_1" + }, + "award_emoji_data":[ ], + "blocking_issues_count":0, + "closed_at":null, + "closed_by":null, + "confidential":false, + "created_at":"2026-01-20T09:57:16.287Z", + "description":"Body task", + "discussion_locked":null, + "downvotes":0, + "due_date":null, + "has_tasks":true, + "id":181476687, + "iid":4, + "imported":false, + "imported_from":"none", + "issue_type":"task", + "labels":[ ], + "merge_requests_count":0, + "milestone":null, + "moved_to_id":null, + "notes_data":[ + { + "author":{ + "avatar_url":"https://secure.gravatar.com/avatar/xxxxxxxx?s=80&d=identicon", + "id":8276902, + "locked":false, + "name":"User Test", + "public_email":"", + "state":"active", + "username":"user_1", + "web_url":"https://gitlab.com/user_1" + }, + "award_emoji_data":[ ], + "body":"assigned to @user_1", + "commands_changes":{ }, + "confidential":false, + "created_at":"2026-01-20T09:57:16.435Z", + "id":3019322655, + "imported":false, + "imported_from":"none", + "internal":false, + "noteable_id":181476687, + "noteable_iid":4, + "noteable_type":"Issue", + "project_id":12345678, + "resolvable":false, + "system":true, + "type":null, + "updated_at":"2026-01-20T09:57:16.444Z" + } + ], + "project_id":12345678, + "references":{ + "full":"user_1/tmp#4", + "relative":"#4", + "short":"#4" + }, + "service_desk_reply_to":null, + "severity":"UNKNOWN", + "state":"opened", + "task_completion_status":{ + "completed_count":0, + "count":0 + }, + "task_status":"0 of 0 checklist items completed", + "time_stats":{ + "human_time_estimate":null, + "human_total_time_spent":null, + "time_estimate":0, + "total_time_spent":0 + }, + "title":"Example task", + "type":"TASK", + "updated_at":"2026-01-20T09:57:16.287Z", + "upvotes":0, + "user_notes_count":0, + "web_url":"https://gitlab.com/user_1/tmp/-/work_items/4" + }, + "origin":"https://gitlab.com/user_1/tmp", + "perceval_version":"1.4.4", + "search_fields":{ + "groups":null, + "iid":4, + "item_id":"181476687", + "owner":"user_1", + "project":"tmp" + }, + "tag":"https://gitlab.com/user_1/tmp", + "timestamp":1.768904307043856E9, + "updated_on":1.768903036287E9, + "uuid":"9d47dbbd2df9579269d1d409548d04c51cf1f0b9" + }, + { + "backend_name":"GitLab", + "backend_version":"1.0.0", + "category":"issue", + "classified_fields_filtered":null, + "data":{ + "_links":{ + "award_emoji":"https://gitlab.com/api/v4/projects/12345678/issues/1/award_emoji", + "closed_as_duplicate_of":null, + "notes":"https://gitlab.com/api/v4/projects/12345678/issues/1/notes", + "project":"https://gitlab.com/api/v4/projects/12345678", + "self":"https://gitlab.com/api/v4/projects/12345678/issues/1" + }, + "assignee":{ + "avatar_url":"https://secure.gravatar.com/avatar/xxxxxxxx?s=80&d=identicon", + "id":8276902, + "locked":false, + "name":"User Test", + "public_email":"", + "state":"active", + "username":"user_1", + "web_url":"https://gitlab.com/user_1" + }, + "assignees":[ + { + "avatar_url":"https://secure.gravatar.com/avatar/xxxxxxxx?s=80&d=identicon", + "id":8276902, + "locked":false, + "name":"User Test", + "public_email":"", + "state":"active", + "username":"user_1", + "web_url":"https://gitlab.com/user_1" + } + ], + "author":{ + "avatar_url":"https://secure.gravatar.com/avatar/xxxxxxxx?s=80&d=identicon", + "id":8276902, + "locked":false, + "name":"User Test", + "public_email":"", + "state":"active", + "username":"user_1", + "web_url":"https://gitlab.com/user_1" + }, + "award_emoji_data":[ ], + "blocking_issues_count":0, + "closed_at":null, + "closed_by":null, + "confidential":false, + "created_at":"2026-01-20T09:55:57.188Z", + "description":"Body issue 1", + "discussion_locked":null, + "downvotes":0, + "due_date":null, + "has_tasks":true, + "id":181476619, + "iid":1, + "imported":false, + "imported_from":"none", + "issue_type":"issue", + "labels":[ ], + "merge_requests_count":0, + "milestone":null, + "moved_to_id":null, + "notes_data":[ + { + "author":{ + "avatar_url":"https://secure.gravatar.com/avatar/xxxxxxxx?s=80&d=identicon", + "id":8276902, + "locked":false, + "name":"User Test", + "public_email":"", + "state":"active", + "username":"user_1", + "web_url":"https://gitlab.com/user_1" + }, + "award_emoji_data":[ ], + "body":"assigned to @user_1", + "commands_changes":{ }, + "confidential":false, + "created_at":"2026-01-20T09:55:57.529Z", + "id":3019317146, + "imported":false, + "imported_from":"none", + "internal":false, + "noteable_id":181476619, + "noteable_iid":1, + "noteable_type":"Issue", + "project_id":12345678, + "resolvable":false, + "system":true, + "type":null, + "updated_at":"2026-01-20T09:55:57.536Z" + }, + { + "author":{ + "avatar_url":"https://secure.gravatar.com/avatar/xxxxxxxx?s=80&d=identicon", + "id":8276902, + "locked":false, + "name":"User Test", + "public_email":"", + "state":"active", + "username":"user_1", + "web_url":"https://gitlab.com/user_1" + }, + "award_emoji_data":[ ], + "body":"This is a comment", + "commands_changes":{ }, + "confidential":false, + "created_at":"2026-01-20T10:18:14.789Z", + "id":3019389027, + "imported":false, + "imported_from":"none", + "internal":false, + "noteable_id":181476619, + "noteable_iid":1, + "noteable_type":"Issue", + "project_id":12345678, + "resolvable":false, + "system":false, + "type":null, + "updated_at":"2026-01-20T10:18:14.789Z" + } + ], + "project_id":12345678, + "references":{ + "full":"user_1/tmp#1", + "relative":"#1", + "short":"#1" + }, + "service_desk_reply_to":null, + "severity":"UNKNOWN", + "state":"opened", + "task_completion_status":{ + "completed_count":0, + "count":0 + }, + "task_status":"0 of 0 checklist items completed", + "time_stats":{ + "human_time_estimate":null, + "human_total_time_spent":null, + "time_estimate":0, + "total_time_spent":0 + }, + "title":"Example issue 1", + "type":"ISSUE", + "updated_at":"2026-01-20T10:18:14.879Z", + "upvotes":0, + "user_notes_count":1, + "web_url":"https://gitlab.com/user_1/tmp/-/issues/1" + }, + "origin":"https://gitlab.com/user_1/tmp", + "perceval_version":"1.4.4", + "search_fields":{ + "groups":null, + "iid":1, + "item_id":"181476619", + "owner":"user_1", + "project":"tmp" + }, + "tag":"https://gitlab.com/user_1/tmp", + "timestamp":1.768904308924827E9, + "updated_on":1.768904294879E9, + "uuid":"753f641525292d18618695f3c4ea7d36a9253fce" + } +] \ No newline at end of file diff --git a/tests/unit/scheduler/test_task_eventizer.py b/tests/unit/scheduler/test_task_eventizer.py index 4d5ed13..0acbf86 100644 --- a/tests/unit/scheduler/test_task_eventizer.py +++ b/tests/unit/scheduler/test_task_eventizer.py @@ -22,6 +22,7 @@ import pickle import shutil import tempfile +from unittest.mock import patch import rq import perceval.backend @@ -44,7 +45,7 @@ def tearDown(self): shutil.rmtree(self.tmp_path) super().tearDown() - def test_job(self): + def test_git_job(self): """Test if events are generated using the Git backend""" job_args = { @@ -205,6 +206,69 @@ def test_job(self): self.assertEqual(event["type"], expected[i][1]) self.assertEqual(event["source"], "http://example.com/") + @patch("grimoirelab.core.scheduler.tasks.chronicler.perceval.backend.BackendItemsGenerator") + def test_gitlab_job(self, mock_backend_items_generator): + """Test if events are generated using the GitLab backend""" + + with open(os.path.join(self.dir, "data/gitlab_issues.json"), "r") as f: + gitlab_issues_data = json.load(f) + + # Return value under .items + mock_backend_items_generator.return_value.items = gitlab_issues_data + + job_args = { + "datasource_type": "gitlab", + "datasource_category": "issue", + "events_stream": "events", + "stream_max_length": 500, + "job_args": { + "owner": "user_1", + "repository": "tmp", + "from_date": "2020-01-01T00:00:00Z", + }, + } + + q = rq.Queue("test-queue", job_class=GrimoireLabJob, connection=self.conn, is_async=False) + job = q.enqueue( + f=chronicler_job, + result_ttl=100, + job_timeout=120, + job_id="chonicler-gitlab", + **job_args, + ) + + # Use job._result instead of job.return_value() because + # the mocked backend fail to produce a return value + result = job._result + + # Check job result + self.assertEqual(result.job_id, job.get_id()) + self.assertEqual(result.backend, "gitlab") + self.assertEqual(result.category, "issue") + + expected = [ + ("cbd19489e7812e32f5f26473e202af1a892c0a65", "org.grimoirelab.events.gitlab.issue"), + ("a1d20ee20e89c4ab33e4a17725ec4f002e9882ec", "org.grimoirelab.events.gitlab.author"), + ("a99cf7a6861409aba3c32c28558cbc97db4a02bf", "org.grimoirelab.events.gitlab.closed_by"), + ("8ea9a635f395f8dadafd932c59b1e42e754eefd0", "org.grimoirelab.events.gitlab.issue"), + ("4a9a2876cc746cfc993cf659a1702ccaca331c27", "org.grimoirelab.events.gitlab.author"), + ("9d47dbbd2df9579269d1d409548d04c51cf1f0b9", "org.grimoirelab.events.gitlab.issue"), + ("8ab2b765e2b5322e1b6186ff3fc38e47dc87e65e", "org.grimoirelab.events.gitlab.author"), + ("be4e5be81c88ad0c9735f6a591a265373308405b", "org.grimoirelab.events.gitlab.assignee"), + ("753f641525292d18618695f3c4ea7d36a9253fce", "org.grimoirelab.events.gitlab.issue"), + ("4c7134547d2f6d4dbdb8231222e35606095db45e", "org.grimoirelab.events.gitlab.author"), + ("16015c50f2e11af6cc2c8bfcf5df4494abd1b1e0", "org.grimoirelab.events.gitlab.assignee"), + ] + + # Check generated events + events = self.conn.xread({"events": b"0-0"}, count=None, block=0) + events = [json.loads(e[1][b"data"]) for e in events[0][1]] + self.assertEqual(len(events), 11) + for i, event in enumerate(events): + self.assertEqual(event["source"], "https://gitlab.com/user_1/tmp") + self.assertEqual(event["id"], expected[i][0]) + self.assertEqual(event["type"], expected[i][1]) + def test_job_no_result(self): """Execute a job that will not produce any results""" diff --git a/ui/src/components/FormDialog.vue b/ui/src/components/FormDialog.vue index 170479c..81716ee 100644 --- a/ui/src/components/FormDialog.vue +++ b/ui/src/components/FormDialog.vue @@ -19,7 +19,7 @@