diff --git a/app/jobs/inactive_urn_list_api_sync_job.rb b/app/jobs/inactive_urn_list_api_sync_job.rb index 90778ab37..efbdc85c0 100644 --- a/app/jobs/inactive_urn_list_api_sync_job.rb +++ b/app/jobs/inactive_urn_list_api_sync_job.rb @@ -1,6 +1,6 @@ class InactiveUrnListApiSyncJob < ApplicationJob def perform - rows = UrnLists::ApiClient.new.fetch_inactive_customers + rows = UrnLists::ApiClient.new.fetch_inactive_rows UrnLists::ImportInactiveCustomers.new(rows: rows).call end diff --git a/app/jobs/urn_list_api_sync_job.rb b/app/jobs/urn_list_api_sync_job.rb index 3e0a0c1f5..8a012c7c4 100644 --- a/app/jobs/urn_list_api_sync_job.rb +++ b/app/jobs/urn_list_api_sync_job.rb @@ -2,7 +2,7 @@ class UrnListApiSyncJob < ApplicationJob def perform urn_list = UrnList.create!(aasm_state: :pending, source: 'api_import') - rows = UrnLists::ApiClient.new.fetch_customers + rows = UrnLists::ApiClient.new.fetch_rows count = UrnLists::ImportCustomers.new(rows: rows).call urn_list.update!( diff --git a/app/services/urn_lists/api_client.rb b/app/services/urn_lists/api_client.rb index 9f80f7fcb..0a8cb7250 100644 --- a/app/services/urn_lists/api_client.rb +++ b/app/services/urn_lists/api_client.rb @@ -6,45 +6,71 @@ module UrnLists class ApiClient class ApiError < StandardError; end - def fetch_customers - token = fetch_access_token - fetch_urn_list(token) + TOP_COUNT = 1000 + + def fetch_rows + fetch_paginated_rows( + base_url: active_urns_url, + params: { + 'api-version' => '2016-10-01', + 'sp' => '/triggers/manual/run', + 'sv' => '1.0', + 'filter' => "Published eq 'True'" + }, + error_message: 'Failed to fetch URN list' + ) end - def fetch_inactive_customers - token = fetch_access_token - fetch_inactive_urn_list(token) + def fetch_inactive_rows + fetch_paginated_rows( + base_url: inactive_urns_url, + params: { + 'api-version' => '2016-10-01', + 'sp' => '/triggers/manual/run', + 'sv' => '1.0' + }, + error_message: 'Failed to fetch inactive URN list' + ) end private - def fetch_access_token - uri = URI.parse(ENV.fetch('MDM_API_TOKEN_URL')) + def fetch_paginated_rows(base_url:, params:, error_message:) + token = fetch_access_token - response = Net::HTTP.post_form(uri, { - grant_type: 'client_credentials', - client_id: ENV.fetch('MDM_API_CLIENT_ID'), - client_secret: ENV.fetch('MDM_API_CLIENT_SECRET'), - scope: ENV.fetch('MDM_API_SCOPE') - }) + all_rows = [] + skip = 0 - raise ApiError, "Failed to fetch access token: #{response.code}" unless response.is_a?(Net::HTTPSuccess) + loop do + rows = fetch_page( + token: token, + base_url: base_url, + params: params, + top_count: TOP_COUNT, + skip: skip, + error_message: error_message + ) - body = JSON.parse(response.body) - body.fetch('access_token') - end + break if rows.empty? - def fetch_urn_list(token) - base_url = 'https://apim.crowncommercial.gov.uk/website-data/manual/paths/invoke/%5Batt%5D.%5Bvw_RMIActiveURNList%5D/' - params = { - 'api-version' => '2016-10-01', - 'sp' => '/triggers/manual/run', - 'sv' => '1.0', - 'filter' => "Published eq 'True'" - } + all_rows.concat(rows) + break if rows.size < TOP_COUNT + skip += TOP_COUNT + end + + all_rows + end + + # rubocop:disable Metrics/ParameterLists + def fetch_page(token:, base_url:, params:, top_count:, skip:, error_message:) uri = URI(base_url) - uri.query = URI.encode_www_form(params) + uri.query = URI.encode_www_form( + params.merge( + 'TopCount' => top_count, + 'SkipCount' => skip + ) + ) request = Net::HTTP::Get.new(uri.to_s) request['Authorization'] = "Bearer #{token}" @@ -54,37 +80,28 @@ def fetch_urn_list(token) http.request(request) end - raise ApiError, "Failed to fetch URN list: #{response.code}" unless response.is_a?(Net::HTTPSuccess) + raise ApiError, "#{error_message}: #{response.code}" unless response.is_a?(Net::HTTPSuccess) rows = JSON.parse(response.body) validate_rows!(rows) rows end + # rubocop:enable Metrics/ParameterLists - def fetch_inactive_urn_list(token) - base_url = 'https://apim.crowncommercial.gov.uk/website-data/manual/paths/invoke/%5Batt%5D.%5Bvw_RMIInactiveURNList%5D/' - params = { - 'api-version' => '2016-10-01', - 'sp' => '/triggers/manual/run', - 'sv' => '1.0' - } - - uri = URI(base_url) - uri.query = URI.encode_www_form(params) - - request = Net::HTTP::Get.new(uri.to_s) - request['Authorization'] = "Bearer #{token}" - request['Accept'] = 'application/json' + def fetch_access_token + uri = URI.parse(ENV.fetch('MDM_API_TOKEN_URL')) - response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https') do |http| - http.request(request) - end + response = Net::HTTP.post_form(uri, { + grant_type: 'client_credentials', + client_id: ENV.fetch('MDM_API_CLIENT_ID'), + client_secret: ENV.fetch('MDM_API_CLIENT_SECRET'), + scope: ENV.fetch('MDM_API_SCOPE') + }) - raise ApiError, "Failed to fetch inactive URN list: #{response.code}" unless response.is_a?(Net::HTTPSuccess) + raise ApiError, "Failed to fetch access token: #{response.code}" unless response.is_a?(Net::HTTPSuccess) - rows = JSON.parse(response.body) - validate_rows!(rows) - rows + body = JSON.parse(response.body) + body.fetch('access_token') end def validate_rows!(rows) @@ -92,5 +109,13 @@ def validate_rows!(rows) raise ApiError, 'Invalid URN list format: expected an array of objects' end + + def active_urns_url + 'https://apim.crowncommercial.gov.uk/website-data/manual/paths/invoke/%5Batt%5D.%5Bvw_RMIActiveURNList%5D/' + end + + def inactive_urns_url + 'https://apim.crowncommercial.gov.uk/website-data/manual/paths/invoke/%5Batt%5D.%5Bvw_RMIInactiveURNList%5D/' + end end end diff --git a/spec/jobs/urn_list_api_sync_job_spec.rb b/spec/jobs/urn_list_api_sync_job_spec.rb index d8c9f9b52..48484bc4b 100644 --- a/spec/jobs/urn_list_api_sync_job_spec.rb +++ b/spec/jobs/urn_list_api_sync_job_spec.rb @@ -23,7 +23,7 @@ ] end - let(:api_client_service) { double('UrnLists::ApiClient', fetch_customers: rows) } + let(:api_client_service) { double('UrnLists::ApiClient', fetch_rows: rows) } let(:import_customers_service) { double('UrnLists::ImportCustomers', call: rows.count) } before do @@ -36,7 +36,7 @@ described_class.perform_now end.to change(UrnList, :count).by(1) - expect(api_client_service).to have_received(:fetch_customers) + expect(api_client_service).to have_received(:fetch_rows) expect(import_customers_service).to have_received(:call) urn_list = UrnList.last @@ -47,7 +47,7 @@ end it 'marks the urn list as failed when the api call fails' do - allow(api_client_service).to receive(:fetch_customers).and_raise(StandardError.new('token failed')) + allow(api_client_service).to receive(:fetch_rows).and_raise(StandardError.new('token failed')) expect do described_class.perform_now diff --git a/spec/services/urn_lists/api_client_spec.rb b/spec/services/urn_lists/api_client_spec.rb index b6aed6124..d716fe24e 100644 --- a/spec/services/urn_lists/api_client_spec.rb +++ b/spec/services/urn_lists/api_client_spec.rb @@ -1,7 +1,9 @@ require 'rails_helper' RSpec.describe UrnLists::ApiClient do - describe '#fetch_customers' do + describe '#fetch_rows' do + let(:top_count) { described_class::TOP_COUNT } + before do stub_request(:post, 'https://example.com/oauth/token') .with( @@ -9,10 +11,10 @@ 'grant_type' => 'client_credentials', 'scope' => 'test_scope' }, headers: { 'Accept' => '*/*', - 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', - 'Content-Type' => 'application/x-www-form-urlencoded', - 'Host' => 'example.com', - 'User-Agent' => 'Ruby' + 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', + 'Content-Type' => 'application/x-www-form-urlencoded', + 'Host' => 'example.com', + 'User-Agent' => 'Ruby' } ) .to_return( @@ -21,13 +23,13 @@ headers: { 'Content-Type' => 'application/json' } ) - stub_request(:get, "https://apim.crowncommercial.gov.uk/website-data/manual/paths/invoke/%5Batt%5D.%5Bvw_RMIActiveURNList%5D/?api-version=2016-10-01&filter=Published%20eq%20'True'&sp=/triggers/manual/run&sv=1.0") + stub_request(:get, "https://apim.crowncommercial.gov.uk/website-data/manual/paths/invoke/%5Batt%5D.%5Bvw_RMIActiveURNList%5D/?SkipCount=0&TopCount=1000&api-version=2016-10-01&filter=Published%20eq%20'True'&sp=/triggers/manual/run&sv=1.0") .with( headers: { 'Accept' => 'application/json', - 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', - 'Authorization' => 'Bearer abc123', - 'User-Agent' => 'Ruby' + 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', + 'Authorization' => 'Bearer abc123', + 'User-Agent' => 'Ruby' } ) .to_return( @@ -53,7 +55,7 @@ it 'fetches and returns customer data' do client = described_class.new - customers = client.fetch_customers + customers = client.fetch_rows expect(customers.size).to eq(1) expect(customers.first['urn']).to eq(10009655) @@ -62,5 +64,71 @@ expect(customers.first['sector']).to eq('central_government') expect(customers.first['published']).to eq(true) end + + context 'when the API returns multiple pages' do + let(:first_page_rows) do + Array.new(top_count) do |i| + { + urn: 10009655 + i, + name: "Customer #{i}", + postcode: 'L3 9PP', + sector: 'central_government', + published: true + } + end + end + + let(:second_page_rows) do + [ + { + urn: 10009655 + top_count, + name: "Customer #{top_count}", + postcode: 'L3 9PP', + sector: 'central_government', + published: true + } + ] + end + + before do + stub_request(:get, "https://apim.crowncommercial.gov.uk/website-data/manual/paths/invoke/%5Batt%5D.%5Bvw_RMIActiveURNList%5D/?TopCount=#{top_count}&SkipCount=0&api-version=2016-10-01&filter=Published%20eq%20'True'&sp=/triggers/manual/run&sv=1.0") + .with( + headers: { + 'Accept' => 'application/json', + 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', + 'Authorization' => 'Bearer abc123', + 'User-Agent' => 'Ruby' + } + ) + .to_return( + status: 200, + body: first_page_rows.to_json, + headers: { 'Content-Type' => 'application/json' } + ) + + stub_request(:get, "https://apim.crowncommercial.gov.uk/website-data/manual/paths/invoke/%5Batt%5D.%5Bvw_RMIActiveURNList%5D/?TopCount=#{top_count}&SkipCount=#{top_count}&api-version=2016-10-01&filter=Published%20eq%20'True'&sp=/triggers/manual/run&sv=1.0") + .with( + headers: { + 'Accept' => 'application/json', + 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', + 'Authorization' => 'Bearer abc123', + 'User-Agent' => 'Ruby' + } + ) + .to_return( + status: 200, + body: second_page_rows.to_json, + headers: { 'Content-Type' => 'application/json' } + ) + end + + it 'fetches each page and returns combined rows' do + rows = described_class.new.fetch_rows + + expect(rows.count).to eq(top_count + 1) + expect(rows.first['urn']).to eq(10009655) + expect(rows.last['urn']).to eq(10009655 + top_count) + end + end end end