From b7586f0f658b4040a4f27a15b7fbafd8c47da23a Mon Sep 17 00:00:00 2001 From: Reem Ibrahim Date: Fri, 29 May 2026 17:05:53 +0100 Subject: [PATCH 1/2] updated sync jobs and spec --- app/jobs/inactive_urn_list_api_sync_job.rb | 2 +- app/jobs/urn_list_api_sync_job.rb | 2 +- app/services/urn_lists/api_client.rb | 123 ++++++++++++--------- spec/jobs/urn_list_api_sync_job_spec.rb | 6 +- spec/services/urn_lists/api_client_spec.rb | 89 +++++++++++++-- 5 files changed, 157 insertions(+), 65 deletions(-) diff --git a/app/jobs/inactive_urn_list_api_sync_job.rb b/app/jobs/inactive_urn_list_api_sync_job.rb index 90778ab37..efbdc85c0 100644 --- a/app/jobs/inactive_urn_list_api_sync_job.rb +++ b/app/jobs/inactive_urn_list_api_sync_job.rb @@ -1,6 +1,6 @@ class InactiveUrnListApiSyncJob < ApplicationJob def perform - rows = UrnLists::ApiClient.new.fetch_inactive_customers + rows = UrnLists::ApiClient.new.fetch_inactive_rows UrnLists::ImportInactiveCustomers.new(rows: rows).call end diff --git a/app/jobs/urn_list_api_sync_job.rb b/app/jobs/urn_list_api_sync_job.rb index 3e0a0c1f5..8a012c7c4 100644 --- a/app/jobs/urn_list_api_sync_job.rb +++ b/app/jobs/urn_list_api_sync_job.rb @@ -2,7 +2,7 @@ class UrnListApiSyncJob < ApplicationJob def perform urn_list = UrnList.create!(aasm_state: :pending, source: 'api_import') - rows = UrnLists::ApiClient.new.fetch_customers + rows = UrnLists::ApiClient.new.fetch_rows count = UrnLists::ImportCustomers.new(rows: rows).call urn_list.update!( diff --git a/app/services/urn_lists/api_client.rb b/app/services/urn_lists/api_client.rb index 9f80f7fcb..5dffa3e2c 100644 --- a/app/services/urn_lists/api_client.rb +++ b/app/services/urn_lists/api_client.rb @@ -6,71 +6,70 @@ module UrnLists class ApiClient class ApiError < StandardError; end - def fetch_customers - token = fetch_access_token - fetch_urn_list(token) + TOP_COUNT = 1000 + + def fetch_rows + fetch_paginated_rows( + base_url: active_urns_url, + params: { + 'api-version' => '2016-10-01', + 'sp' => '/triggers/manual/run', + 'sv' => '1.0', + 'filter' => "Published eq 'True'" + }, + error_message: 'Failed to fetch URN list' + ) end - def fetch_inactive_customers - token = fetch_access_token - fetch_inactive_urn_list(token) + def fetch_inactive_rows + fetch_paginated_rows( + base_url: inactive_urns_url, + params: { + 'api-version' => '2016-10-01', + 'sp' => '/triggers/manual/run', + 'sv' => '1.0' + }, + error_message: 'Failed to fetch inactive URN list' + ) end private - def fetch_access_token - uri = URI.parse(ENV.fetch('MDM_API_TOKEN_URL')) - - response = Net::HTTP.post_form(uri, { - grant_type: 'client_credentials', - client_id: ENV.fetch('MDM_API_CLIENT_ID'), - client_secret: ENV.fetch('MDM_API_CLIENT_SECRET'), - scope: ENV.fetch('MDM_API_SCOPE') - }) - - raise ApiError, "Failed to fetch access token: #{response.code}" unless response.is_a?(Net::HTTPSuccess) + def fetch_paginated_rows(base_url:, params:, error_message:) + token = fetch_access_token - body = JSON.parse(response.body) - body.fetch('access_token') - end + all_rows = [] + skip = 0 - def fetch_urn_list(token) - base_url = 'https://apim.crowncommercial.gov.uk/website-data/manual/paths/invoke/%5Batt%5D.%5Bvw_RMIActiveURNList%5D/' - params = { - 'api-version' => '2016-10-01', - 'sp' => '/triggers/manual/run', - 'sv' => '1.0', - 'filter' => "Published eq 'True'" - } + loop do + rows = fetch_page( + token: token, + base_url: base_url, + params: params, + top_count: TOP_COUNT, + skip: skip, + error_message: error_message + ) - uri = URI(base_url) - uri.query = URI.encode_www_form(params) + break if rows.empty? - request = Net::HTTP::Get.new(uri.to_s) - request['Authorization'] = "Bearer #{token}" - request['Accept'] = 'application/json' + all_rows.concat(rows) + break if rows.size < TOP_COUNT - response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https') do |http| - http.request(request) + skip += TOP_COUNT end - raise ApiError, "Failed to fetch URN list: #{response.code}" unless response.is_a?(Net::HTTPSuccess) - - rows = JSON.parse(response.body) - validate_rows!(rows) - rows + all_rows end - def fetch_inactive_urn_list(token) - base_url = 'https://apim.crowncommercial.gov.uk/website-data/manual/paths/invoke/%5Batt%5D.%5Bvw_RMIInactiveURNList%5D/' - params = { - 'api-version' => '2016-10-01', - 'sp' => '/triggers/manual/run', - 'sv' => '1.0' - } - + def fetch_page(token:, base_url:, params:, top_count:, skip:, error_message:) uri = URI(base_url) - uri.query = URI.encode_www_form(params) + uri.query = URI.encode_www_form( + params.merge( + 'TopCount' => top_count, + 'SkipCount' => skip + ) + ) request = Net::HTTP::Get.new(uri.to_s) request['Authorization'] = "Bearer #{token}" @@ -80,11 +79,27 @@ def fetch_inactive_urn_list(token) http.request(request) end - raise ApiError, "Failed to fetch inactive URN list: #{response.code}" unless response.is_a?(Net::HTTPSuccess) + raise ApiError, "#{error_message}: #{response.code}" unless response.is_a?(Net::HTTPSuccess) rows = JSON.parse(response.body) validate_rows!(rows) rows + end + + def fetch_access_token + uri = URI.parse(ENV.fetch('MDM_API_TOKEN_URL')) + + response = Net::HTTP.post_form(uri, { + grant_type: 'client_credentials', + client_id: ENV.fetch('MDM_API_CLIENT_ID'), + client_secret: ENV.fetch('MDM_API_CLIENT_SECRET'), + scope: ENV.fetch('MDM_API_SCOPE') + }) + + raise ApiError, "Failed to fetch access token: #{response.code}" unless response.is_a?(Net::HTTPSuccess) + + body = JSON.parse(response.body) + body.fetch('access_token') end def validate_rows!(rows) @@ -92,5 +107,13 @@ def validate_rows!(rows) raise ApiError, 'Invalid URN list format: expected an array of objects' end + + def active_urns_url + 'https://apim.crowncommercial.gov.uk/website-data/manual/paths/invoke/%5Batt%5D.%5Bvw_RMIActiveURNList%5D/' + end + + def inactive_urns_url + 'https://apim.crowncommercial.gov.uk/website-data/manual/paths/invoke/%5Batt%5D.%5Bvw_RMIInactiveURNList%5D/' + end end end diff --git a/spec/jobs/urn_list_api_sync_job_spec.rb b/spec/jobs/urn_list_api_sync_job_spec.rb index d8c9f9b52..48484bc4b 100644 --- a/spec/jobs/urn_list_api_sync_job_spec.rb +++ b/spec/jobs/urn_list_api_sync_job_spec.rb @@ -23,7 +23,7 @@ ] end - let(:api_client_service) { double('UrnLists::ApiClient', fetch_customers: rows) } + let(:api_client_service) { double('UrnLists::ApiClient', fetch_rows: rows) } let(:import_customers_service) { double('UrnLists::ImportCustomers', call: rows.count) } before do @@ -36,7 +36,7 @@ described_class.perform_now end.to change(UrnList, :count).by(1) - expect(api_client_service).to have_received(:fetch_customers) + expect(api_client_service).to have_received(:fetch_rows) expect(import_customers_service).to have_received(:call) urn_list = UrnList.last @@ -47,7 +47,7 @@ end it 'marks the urn list as failed when the api call fails' do - allow(api_client_service).to receive(:fetch_customers).and_raise(StandardError.new('token failed')) + allow(api_client_service).to receive(:fetch_rows).and_raise(StandardError.new('token failed')) expect do described_class.perform_now diff --git a/spec/services/urn_lists/api_client_spec.rb b/spec/services/urn_lists/api_client_spec.rb index b6aed6124..424805336 100644 --- a/spec/services/urn_lists/api_client_spec.rb +++ b/spec/services/urn_lists/api_client_spec.rb @@ -1,7 +1,9 @@ require 'rails_helper' RSpec.describe UrnLists::ApiClient do - describe '#fetch_customers' do + describe '#fetch_rows' do + let(:top_count) { described_class::TOP_COUNT } + before do stub_request(:post, 'https://example.com/oauth/token') .with( @@ -9,10 +11,10 @@ 'grant_type' => 'client_credentials', 'scope' => 'test_scope' }, headers: { 'Accept' => '*/*', - 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', - 'Content-Type' => 'application/x-www-form-urlencoded', - 'Host' => 'example.com', - 'User-Agent' => 'Ruby' + 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', + 'Content-Type' => 'application/x-www-form-urlencoded', + 'Host' => 'example.com', + 'User-Agent' => 'Ruby' } ) .to_return( @@ -21,13 +23,13 @@ headers: { 'Content-Type' => 'application/json' } ) - stub_request(:get, "https://apim.crowncommercial.gov.uk/website-data/manual/paths/invoke/%5Batt%5D.%5Bvw_RMIActiveURNList%5D/?api-version=2016-10-01&filter=Published%20eq%20'True'&sp=/triggers/manual/run&sv=1.0") + stub_request(:get, "https://apim.crowncommercial.gov.uk/website-data/manual/paths/invoke/%5Batt%5D.%5Bvw_RMIActiveURNList%5D/?SkipCount=0&TopCount=1000&api-version=2016-10-01&filter=Published%20eq%20'True'&sp=/triggers/manual/run&sv=1.0") .with( headers: { 'Accept' => 'application/json', - 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', - 'Authorization' => 'Bearer abc123', - 'User-Agent' => 'Ruby' + 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', + 'Authorization' => 'Bearer abc123', + 'User-Agent' => 'Ruby' } ) .to_return( @@ -53,7 +55,7 @@ it 'fetches and returns customer data' do client = described_class.new - customers = client.fetch_customers + customers = client.fetch_rows expect(customers.size).to eq(1) expect(customers.first['urn']).to eq(10009655) @@ -62,5 +64,72 @@ expect(customers.first['sector']).to eq('central_government') expect(customers.first['published']).to eq(true) end + + context 'when the API returns multiple pages' do + let(:first_page_rows) do + Array.new(top_count) do |i| + { + urn: 10009655 + i, + name: "Customer #{i}", + postcode: 'L3 9PP', + sector: 'central_government', + published: true + } + end + end + + let(:second_page_rows) do + [ + { + urn: 10009655 + top_count, + name: "Customer #{top_count}", + postcode: 'L3 9PP', + sector: 'central_government', + published: true + } + ] + end + + before do + stub_request(:get, "https://apim.crowncommercial.gov.uk/website-data/manual/paths/invoke/%5Batt%5D.%5Bvw_RMIActiveURNList%5D/?TopCount=#{top_count}&SkipCount=0&api-version=2016-10-01&filter=Published%20eq%20'True'&sp=/triggers/manual/run&sv=1.0") + .with( + headers: { + 'Accept' => 'application/json', + 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', + 'Authorization' => 'Bearer abc123', + 'User-Agent' => 'Ruby' + } + ) + .to_return( + status: 200, + body: first_page_rows.to_json, + headers: { 'Content-Type' => 'application/json' } + ) + + stub_request(:get, "https://apim.crowncommercial.gov.uk/website-data/manual/paths/invoke/%5Batt%5D.%5Bvw_RMIActiveURNList%5D/?TopCount=#{top_count}&SkipCount=#{top_count}&api-version=2016-10-01&filter=Published%20eq%20'True'&sp=/triggers/manual/run&sv=1.0") + .with( + headers: { + 'Accept' => 'application/json', + 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', + 'Authorization' => 'Bearer abc123', + 'User-Agent' => 'Ruby' + } + ) + .to_return( + status: 200, + body: second_page_rows.to_json, + headers: { 'Content-Type' => 'application/json' } + ) + + end + + it 'fetches each page and returns combined rows' do + rows = described_class.new.fetch_rows + + expect(rows.count).to eq(top_count + 1) + expect(rows.first['urn']).to eq(10009655) + expect(rows.last['urn']).to eq(10009655 + top_count) + end + end end end From f8a42aec5d3a519cc9aee5424d541bdc301a8725 Mon Sep 17 00:00:00 2001 From: Reem Ibrahim Date: Fri, 29 May 2026 17:08:45 +0100 Subject: [PATCH 2/2] rubocop --- app/services/urn_lists/api_client.rb | 4 +++- spec/services/urn_lists/api_client_spec.rb | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/app/services/urn_lists/api_client.rb b/app/services/urn_lists/api_client.rb index 5dffa3e2c..0a8cb7250 100644 --- a/app/services/urn_lists/api_client.rb +++ b/app/services/urn_lists/api_client.rb @@ -62,6 +62,7 @@ def fetch_paginated_rows(base_url:, params:, error_message:) all_rows end + # rubocop:disable Metrics/ParameterLists def fetch_page(token:, base_url:, params:, top_count:, skip:, error_message:) uri = URI(base_url) uri.query = URI.encode_www_form( @@ -84,7 +85,8 @@ def fetch_page(token:, base_url:, params:, top_count:, skip:, error_message:) rows = JSON.parse(response.body) validate_rows!(rows) rows - end + end + # rubocop:enable Metrics/ParameterLists def fetch_access_token uri = URI.parse(ENV.fetch('MDM_API_TOKEN_URL')) diff --git a/spec/services/urn_lists/api_client_spec.rb b/spec/services/urn_lists/api_client_spec.rb index 424805336..d716fe24e 100644 --- a/spec/services/urn_lists/api_client_spec.rb +++ b/spec/services/urn_lists/api_client_spec.rb @@ -120,7 +120,6 @@ body: second_page_rows.to_json, headers: { 'Content-Type' => 'application/json' } ) - end it 'fetches each page and returns combined rows' do