diff --git a/Library/Homebrew/github_runner_matrix.rb b/Library/Homebrew/github_runner_matrix.rb index b643acc566441..9ac91ca19fae8 100644 --- a/Library/Homebrew/github_runner_matrix.rb +++ b/Library/Homebrew/github_runner_matrix.rb @@ -64,28 +64,40 @@ def initialize(testing_formulae, deleted_formulae, all_supported:, dependent_mat @dependent_matrix = dependent_matrix @compatible_testing_formulae = T.let({}, T::Hash[GitHubRunner, T::Array[TestRunnerFormula]]) @formulae_with_untested_dependents = T.let({}, T::Hash[GitHubRunner, T::Array[TestRunnerFormula]]) - + @compatible_untested_dependent_names = T.let({}, T::Hash[GitHubRunner, T::Array[String]]) @runners = T.let([], T::Array[GitHubRunner]) generate_runners! freeze end - sig { returns(T::Array[RunnerSpecHash]) } + sig { returns(T::Array[T::Hash[Symbol, T.untyped]]) } def active_runner_specs_hash - runners.select(&:active) - .map(&:spec) - .map(&:to_h) + runners.filter(&:active).flat_map do |r| + Array.new(shard_count = selected_runner_count_for(r)) do |i| + (spec = r.spec.to_h).merge( + name: (shard_count > 1) ? "#{spec[:name]} (shard #{i + 1}/#{shard_count})" : spec[:name], + shard_index: i, + shard_total: shard_count, + ) + end + end end private SELF_HOSTED_LINUX_RUNNER = "linux-self-hosted-1" + DEPS_SHARDING_ENV = "HOMEBREW_DEPS_SHARDING" + DEPS_SHARD_MAX_RUNNERS_ENV = "HOMEBREW_DEPS_SHARD_MAX_RUNNERS" + DEPS_SHARD_BASE_THRESHOLD_ENV = "HOMEBREW_DEPS_SHARD_BASE_THRESHOLD" + DEPS_SHARD_RUNNER_PENALTY_ENV = "HOMEBREW_DEPS_SHARD_RUNNER_PENALTY" # ARM macOS timeout, keep this under 1/2 of GitHub's job execution time limit for self-hosted runners. # https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners/about-self-hosted-runners#usage-limits GITHUB_ACTIONS_LONG_TIMEOUT = 2160 # 36 hours GITHUB_ACTIONS_SHORT_TIMEOUT = 60 - private_constant :SELF_HOSTED_LINUX_RUNNER, :GITHUB_ACTIONS_LONG_TIMEOUT, :GITHUB_ACTIONS_SHORT_TIMEOUT + private_constant :SELF_HOSTED_LINUX_RUNNER, :DEPS_SHARDING_ENV, :DEPS_SHARD_MAX_RUNNERS_ENV, + :DEPS_SHARD_BASE_THRESHOLD_ENV, :DEPS_SHARD_RUNNER_PENALTY_ENV, + :GITHUB_ACTIONS_LONG_TIMEOUT, :GITHUB_ACTIONS_SHORT_TIMEOUT sig { params(arch: Symbol).returns(LinuxRunnerSpec) } def linux_runner_spec(arch) @@ -261,6 +273,62 @@ def generate_runners! @runners.freeze end + sig { params(runner: GitHubRunner).returns(Integer) } + def selected_runner_count_for(runner) + return 1 if !@dependent_matrix || %w[1 true].exclude?(ENV.fetch(DEPS_SHARDING_ENV, "false").downcase) + + max_available_runners = T.must(sharding_integer_env(DEPS_SHARD_MAX_RUNNERS_ENV, runner, 1)) + base_spillover_threshold = sharding_integer_env(DEPS_SHARD_BASE_THRESHOLD_ENV, runner, nil) + additional_runner_reluctance = sharding_integer_env(DEPS_SHARD_RUNNER_PENALTY_ENV, runner, nil) + raise ArgumentError, "#{DEPS_SHARD_MAX_RUNNERS_ENV} must be positive" unless max_available_runners.positive? + raise ArgumentError, "#{DEPS_SHARD_BASE_THRESHOLD_ENV} must be positive" if base_spillover_threshold&.<= 0 + + if additional_runner_reluctance&.negative? + raise ArgumentError, + "#{DEPS_SHARD_RUNNER_PENALTY_ENV} must be non-negative" + end + return 1 if max_available_runners <= 1 || base_spillover_threshold.nil? || additional_runner_reluctance.nil? + + dependent_names = @compatible_untested_dependent_names[runner] ||= compatible_testing_formulae(runner) + .flat_map do |formula| + compatible_untested_dependent_names_for_formula( + formula, runner + ) + end.uniq.sort + dependent_count = dependent_names.count + return 1 if dependent_count.zero? + + runner_count = if additional_runner_reluctance.zero? + dependent_count.to_f / base_spillover_threshold + else + threshold_difference = base_spillover_threshold - additional_runner_reluctance + discriminant = (threshold_difference**2) + (4 * additional_runner_reluctance * dependent_count) + (Math.sqrt(discriminant.to_f) - threshold_difference) / (2 * additional_runner_reluctance) + end + + runner_count.ceil.clamp(1, max_available_runners) + end + + sig { params(base_env_name: String, runner: GitHubRunner, default: T.nilable(Integer)).returns(T.nilable(Integer)) } + def sharding_integer_env(base_env_name, runner, default) + platform = runner.platform.to_s.upcase + arch = runner.arch.to_s.upcase + version = runner.macos_version&.to_sym + version = version.to_s.upcase if version + + [ + ("#{base_env_name}_#{platform}_#{arch}_#{version}" if version), + "#{base_env_name}_#{platform}_#{arch}", + "#{base_env_name}_#{platform}", + base_env_name, + ].compact.each do |env_name| + env_value = ENV.fetch(env_name, nil) + return Integer(env_value, 10) if env_value.present? + end + + default + end + sig { params(runner: GitHubRunner).returns(T::Array[String]) } def testable_formulae(runner) formulae = if @dependent_matrix @@ -301,27 +369,30 @@ def compatible_testing_formulae(runner) sig { params(runner: GitHubRunner).returns(T::Array[TestRunnerFormula]) } def formulae_with_untested_dependents(runner) - @formulae_with_untested_dependents[runner] ||= begin - platform = runner.platform - arch = runner.arch - macos_version = runner.macos_version + @formulae_with_untested_dependents[runner] ||= compatible_testing_formulae(runner).select do |formula| + compatible_untested_dependent_names_for_formula(formula, runner).present? + end + end - compatible_testing_formulae(runner).select do |formula| - compatible_dependents = formula.dependents(platform:, arch:, macos_version: macos_version&.to_sym) - .select do |dependent_f| - Homebrew::SimulateSystem.with(os: platform, arch: Homebrew::SimulateSystem.arch_symbols.fetch(arch)) do - simulated_dependent_f = TestRunnerFormula.new(Formulary.factory(dependent_f.name)) - next false if macos_version && !simulated_dependent_f.compatible_with?(macos_version) + sig { params(formula: TestRunnerFormula, runner: GitHubRunner).returns(T::Array[String]) } + def compatible_untested_dependent_names_for_formula(formula, runner) + platform = runner.platform + arch = runner.arch + macos_version = runner.macos_version - simulated_dependent_f.public_send(:"#{platform}_compatible?") && - simulated_dependent_f.public_send(:"#{arch}_compatible?") - end - end + compatible_dependents = formula.dependents(platform:, arch:, macos_version: macos_version&.to_sym) + .select do |dependent_f| + Homebrew::SimulateSystem.with(os: platform, arch: Homebrew::SimulateSystem.arch_symbols.fetch(arch)) do + simulated_dependent_f = TestRunnerFormula.new(Formulary.factory(dependent_f.name)) + next false if macos_version && !simulated_dependent_f.compatible_with?(macos_version) - # These arrays will generally have been generated by different Formulary caches, - # so we can only compare them by name and not directly. - (compatible_dependents.map(&:name) - @testing_formulae.map(&:name)).present? + simulated_dependent_f.public_send(:"#{platform}_compatible?") && + simulated_dependent_f.public_send(:"#{arch}_compatible?") end end + + # These arrays will generally have been generated by different Formulary caches, + # so we can only compare them by name and not directly. + compatible_dependents.map(&:name) - @testing_formulae.map(&:name) end end diff --git a/Library/Homebrew/test/github_runner_matrix_spec.rb b/Library/Homebrew/test/github_runner_matrix_spec.rb index 3f5dc29fbb0c0..06e85fbdf3401 100644 --- a/Library/Homebrew/test/github_runner_matrix_spec.rb +++ b/Library/Homebrew/test/github_runner_matrix_spec.rb @@ -6,6 +6,7 @@ RSpec.describe GitHubRunnerMatrix, :no_api do before do + allow(ENV).to receive(:[]).and_call_original allow(ENV).to receive(:fetch).and_call_original allow(ENV).to receive(:fetch).with("HOMEBREW_LINUX_RUNNER").and_return("ubuntu-latest") allow(ENV).to receive(:fetch).with("HOMEBREW_MACOS_LONG_TIMEOUT", "false").and_return("false") @@ -292,6 +293,95 @@ matrix = described_class.new([testball], ["deleted"], all_supported: false, dependent_matrix: true) expect(get_runner_names(matrix)).to eq(["Linux x86_64"]) end + + it "shards dependent runners using the most specific sharding settings" do + allow(Homebrew::EnvConfig).to receive(:eval_all?).and_return(true) + allow(Formula).to receive(:all).and_return([ + testball, + testball_depender_linux, + setup_test_runner_formula("testball-depender-linux-two", ["testball", :linux]), + ].map(&:formula)) + allow(ENV).to receive(:[]).with("HOMEBREW_LINUX_RUNNER").and_return("linux-self-hosted-1") + env_fetch_overrides = { + ["HOMEBREW_LINUX_RUNNER", "ubuntu-latest"] => "linux-self-hosted-1", + ["HOMEBREW_DEPS_SHARDING", "false"] => "1", + ["HOMEBREW_DEPS_SHARD_BASE_THRESHOLD_LINUX_X86_64", nil] => "1", + ["HOMEBREW_DEPS_SHARD_BASE_THRESHOLD_LINUX", nil] => "10", + ["HOMEBREW_DEPS_SHARD_RUNNER_PENALTY", nil] => "0", + ["HOMEBREW_DEPS_SHARD_MAX_RUNNERS_LINUX", nil] => "2", + } + allow(ENV).to receive(:fetch).and_wrap_original do |original, key, default = nil| + if env_fetch_overrides.key?([key, default]) + env_fetch_overrides[[key, default]] + else + original.call(key, default) + end + end + matrix = described_class.new([testball], [], all_supported: false, dependent_matrix: true) + + expect(matrix.active_runner_specs_hash) + .to eq([ + { + name: "Linux x86_64 (shard 1/2)", + runner: "linux-self-hosted-1", + container: { + image: "ghcr.io/homebrew/brew:main", + options: "--user=linuxbrew -e GITHUB_ACTIONS_HOMEBREW_SELF_HOSTED", + }, + workdir: "/github/home", + timeout: 2160, + cleanup: true, + testing_formulae: "testball", + shard_index: 0, + shard_total: 2, + }, + { + name: "Linux x86_64 (shard 2/2)", + runner: "linux-self-hosted-1", + container: { + image: "ghcr.io/homebrew/brew:main", + options: "--user=linuxbrew -e GITHUB_ACTIONS_HOMEBREW_SELF_HOSTED", + }, + workdir: "/github/home", + timeout: 2160, + cleanup: true, + testing_formulae: "testball", + shard_index: 1, + shard_total: 2, + }, + ]) + end + + it "counts shared dependents once when sizing shards" do + shared_testing_formula = setup_test_runner_formula("testball-user") + allow(Homebrew::EnvConfig).to receive(:eval_all?).and_return(true) + allow(Formula).to receive(:all).and_return([ + testball, + shared_testing_formula, + setup_test_runner_formula("testball-shared-depender-linux", ["testball", "testball-user", :linux]), + setup_test_runner_formula("testball-unique-depender-linux", ["testball", :linux]), + ].map(&:formula)) + allow(ENV).to receive(:[]).with("HOMEBREW_LINUX_RUNNER").and_return("linux-self-hosted-1") + env_fetch_overrides = { + ["HOMEBREW_LINUX_RUNNER", "ubuntu-latest"] => "linux-self-hosted-1", + ["HOMEBREW_DEPS_SHARDING", "false"] => "1", + ["HOMEBREW_DEPS_SHARD_BASE_THRESHOLD_LINUX", nil] => "1", + ["HOMEBREW_DEPS_SHARD_RUNNER_PENALTY", nil] => "0", + ["HOMEBREW_DEPS_SHARD_MAX_RUNNERS_LINUX", nil] => "3", + } + allow(ENV).to receive(:fetch).and_wrap_original do |original, key, default = nil| + if env_fetch_overrides.key?([key, default]) + env_fetch_overrides[[key, default]] + else + original.call(key, default) + end + end + + matrix = described_class.new([testball, shared_testing_formula], [], all_supported: false, + dependent_matrix: true) + + expect(matrix.active_runner_specs_hash.count).to eq(2) + end end context "when dependent formulae require macOS" do diff --git a/Library/Homebrew/test/test_bot/formulae_dependents_spec.rb b/Library/Homebrew/test/test_bot/formulae_dependents_spec.rb new file mode 100644 index 0000000000000..b2a4ee0bc02b0 --- /dev/null +++ b/Library/Homebrew/test/test_bot/formulae_dependents_spec.rb @@ -0,0 +1,104 @@ +# typed: false +# frozen_string_literal: true + +require "dev-cmd/test-bot" + +RSpec.describe Homebrew::TestBot::FormulaeDependents do + subject(:formulae_dependents) do + described_class.new(tap: nil, git: "git", dry_run: true, fail_fast: false, verbose: false) + end + + let(:args) { double(skip_recursive_dependents?: false) } + + describe "#configure_dependent_sharding!" do + before do + formulae_dependents.instance_variable_set(:@dependent_testing_formulae, %w[testball testball-user]) + allow(Formulary).to receive(:factory).with("testball").and_return(instance_double(Formula)) + allow(Formulary).to receive(:factory).with("testball-user").and_return(instance_double(Formula)) + allow(formulae_dependents).to receive(:skip_recursive_dependents_for).and_return(false) + allow(formulae_dependents).to receive(:dependent_formula_names) + .with("testball", skip_recursive_dependents: false) + .and_return(%w[alpha shared]) + allow(formulae_dependents).to receive(:dependent_formula_names) + .with("testball-user", skip_recursive_dependents: false) + .and_return(%w[beta shared]) + end + + it "deduplicates shared dependents across testing formulae before sharding" do + with_env( + "HOMEBREW_DEPS_SHARD_INDEX" => "0", + "HOMEBREW_DEPS_SHARD_TOTAL" => "2", + ) do + formulae_dependents.send(:configure_dependent_sharding!, args:) + end + + expect(formulae_dependents.instance_variable_get(:@assigned_dependent_formula_names)) + .to eq(Set.new(%w[alpha shared])) + end + + it "assigns dependents across shards without duplicates" do + shard_zero = described_class.new(tap: nil, git: "git", dry_run: true, fail_fast: false, verbose: false) + shard_one = described_class.new(tap: nil, git: "git", dry_run: true, fail_fast: false, verbose: false) + + [shard_zero, shard_one].each do |deps| + deps.instance_variable_set(:@dependent_testing_formulae, %w[testball testball-user]) + allow(deps).to receive(:skip_recursive_dependents_for).and_return(false) + allow(Formulary).to receive(:factory).with("testball").and_return(instance_double(Formula)) + allow(Formulary).to receive(:factory).with("testball-user").and_return(instance_double(Formula)) + allow(deps).to receive(:dependent_formula_names) + .with("testball", skip_recursive_dependents: false) + .and_return(%w[alpha gamma epsilon]) + allow(deps).to receive(:dependent_formula_names) + .with("testball-user", skip_recursive_dependents: false) + .and_return(%w[beta delta]) + end + + with_env("HOMEBREW_DEPS_SHARD_INDEX" => "0", "HOMEBREW_DEPS_SHARD_TOTAL" => "2") do + shard_zero.send(:configure_dependent_sharding!, args:) + end + with_env("HOMEBREW_DEPS_SHARD_INDEX" => "1", "HOMEBREW_DEPS_SHARD_TOTAL" => "2") do + shard_one.send(:configure_dependent_sharding!, args:) + end + + assigned_zero = shard_zero.instance_variable_get(:@assigned_dependent_formula_names) + assigned_one = shard_one.instance_variable_get(:@assigned_dependent_formula_names) + + expect(assigned_zero | assigned_one).to eq(Set.new(%w[alpha beta delta epsilon gamma])) + expect(assigned_zero & assigned_one).to eq(Set.new) + expect(assigned_zero.count - assigned_one.count).to be <= 1 + end + + it "preserves existing behavior for a single shard" do + dependent = instance_double(Formula, full_name: "alpha") + formulae_dependents.instance_variable_set(:@handled_dependent_formula_names, Set["alpha"]) + + with_env("HOMEBREW_DEPS_SHARD_TOTAL" => "1") do + formulae_dependents.send(:configure_dependent_sharding!, args:) + end + + expect(formulae_dependents.instance_variable_get(:@assigned_dependent_formula_names)).to be_nil + expect(formulae_dependents.instance_variable_get(:@handled_dependent_formula_names)).to eq(Set.new) + expect(formulae_dependents.send(:sharded_dependents, [dependent])).to eq([dependent]) + end + + it "tests a shared dependent once when multiple changed formulae reach it on the same shard" do + alpha = instance_double(Formula, full_name: "alpha") + shared = instance_double(Formula, full_name: "shared") + + with_env( + "HOMEBREW_DEPS_SHARD_INDEX" => "0", + "HOMEBREW_DEPS_SHARD_TOTAL" => "2", + ) do + formulae_dependents.send(:configure_dependent_sharding!, args:) + end + + first_formula_dependents = formulae_dependents.send(:sharded_dependents, [alpha, shared]) + first_formula_dependents.each do |dependent| + formulae_dependents.instance_variable_get(:@handled_dependent_formula_names).add(dependent.full_name) + end + + expect(first_formula_dependents).to eq([alpha, shared]) + expect(formulae_dependents.send(:sharded_dependents, [shared])).to eq([]) + end + end +end diff --git a/Library/Homebrew/test_bot/formulae_dependents.rb b/Library/Homebrew/test_bot/formulae_dependents.rb index 064bbb3609f96..07f3a54e6ba10 100644 --- a/Library/Homebrew/test_bot/formulae_dependents.rb +++ b/Library/Homebrew/test_bot/formulae_dependents.rb @@ -4,6 +4,8 @@ module Homebrew module TestBot class FormulaeDependents < TestFormulae + DEPS_SHARD_INDEX_ENV = "HOMEBREW_DEPS_SHARD_INDEX" + DEPS_SHARD_TOTAL_ENV = "HOMEBREW_DEPS_SHARD_TOTAL" sig { params(testing_formulae: T::Array[String]).returns(T::Array[String]) } attr_writer :testing_formulae @@ -24,6 +26,8 @@ def initialize(tap:, git:, dry_run:, fail_fast:, verbose:) @testing_formulae_with_tested_dependents = T.let([], T::Array[String]) @tested_dependents_list = T.let(nil, T.nilable(Pathname)) @dependent_testing_formulae = T.let([], T::Array[String]) + @assigned_dependent_formula_names = T.let(nil, T.nilable(T::Set[String])) + @handled_dependent_formula_names = T.let(Set.new, T::Set[String]) end sig { params(args: Homebrew::Cmd::TestBotCmd::Args).void } @@ -41,6 +45,7 @@ def run!(args:) @tested_dependents_list = Pathname("tested-dependents-#{Utils::Bottles.tag}.txt") @dependent_testing_formulae = sorted_formulae - skipped_or_failed_formulae + configure_dependent_sharding!(args:) install_formulae_if_needed_from_bottles!(installable_bottles, args:) @@ -145,25 +150,34 @@ def dependent_formulae!(formula_name, args:) bottled_dependents.each do |dependent| install_dependent(dependent, testable_dependents, args:) end + return if @assigned_dependent_formula_names.nil? + + (source_dependents + bottled_dependents).each do |dependent| + @handled_dependent_formula_names.add(dependent.full_name) + end end sig { - params(formula: Formula, formula_name: String, args: Homebrew::Cmd::TestBotCmd::Args) - .returns([T::Array[Formula], T::Array[Formula], T::Array[Formula]]) + params(formula: Formula, args: Homebrew::Cmd::TestBotCmd::Args) + .returns(T::Boolean) } - def dependents_for_formula(formula, formula_name, args:) - info_header "Determining dependents..." - + def skip_recursive_dependents_for(formula, args:) # Always skip recursive dependents on Intel. It's really slow. # Also skip recursive dependents on Linux unless it's a Linux-only formula. # # TODO: move to extend/os # rubocop:todo Homebrew/MoveToExtendOS - skip_recursive_dependents = args.skip_recursive_dependents? || - (OS.mac? && Hardware::CPU.intel?) || - (OS.linux? && formula.requirements.exclude?(LinuxRequirement.new)) + args.skip_recursive_dependents? || + (OS.mac? && Hardware::CPU.intel?) || + (OS.linux? && formula.requirements.exclude?(LinuxRequirement.new)) # rubocop:enable Homebrew/MoveToExtendOS + end + sig { + params(formula_name: String, skip_recursive_dependents: T::Boolean) + .returns(T::Array[String]) + } + def dependent_formula_names(formula_name, skip_recursive_dependents:) uses_args = %w[--formula --eval-all] uses_include_test_args = [*uses_args, "--include-test"] uses_include_test_args << "--recursive" unless skip_recursive_dependents @@ -171,7 +185,6 @@ def dependents_for_formula(formula, formula_name, args:) Utils.safe_popen_read("brew", "uses", *uses_include_test_args, formula_name) .split("\n") end - # TODO: Consider handling the following case better. # `foo` has a build dependency on `bar`, and `bar` has a runtime dependency on # `baz`. When testing `baz` with `--build-dependents-from-source`, `foo` is @@ -182,9 +195,21 @@ def dependents_for_formula(formula, formula_name, args:) end dependents.uniq! dependents.sort! + dependents + end + sig { + params(formula: Formula, formula_name: String, args: Homebrew::Cmd::TestBotCmd::Args) + .returns([T::Array[Formula], T::Array[Formula], T::Array[Formula]]) + } + def dependents_for_formula(formula, formula_name, args:) + info_header "Determining dependents..." + + skip_recursive_dependents = skip_recursive_dependents_for(formula, args:) + dependents = dependent_formula_names(formula_name, skip_recursive_dependents:) dependents -= @tested_formulae dependents = dependents.map { |d| Formulary.factory(d) } + dependents = sharded_dependents(dependents) dependents = dependents.zip(dependents.map do |f| if skip_recursive_dependents @@ -256,6 +281,42 @@ def dependents_for_formula(formula, formula_name, args:) [source_dependents, bottled_dependents, testable_dependents] end + sig { params(args: Homebrew::Cmd::TestBotCmd::Args).void } + def configure_dependent_sharding!(args:) + shard_index = (env_value = ENV.fetch(DEPS_SHARD_INDEX_ENV, nil)).blank? ? 0 : Integer(env_value, 10) + shard_total = (env_value = ENV.fetch(DEPS_SHARD_TOTAL_ENV, nil)).blank? ? 1 : Integer(env_value, 10) + raise ArgumentError, "#{DEPS_SHARD_TOTAL_ENV} must be positive" unless shard_total.positive? + raise ArgumentError, "#{DEPS_SHARD_INDEX_ENV} must be between 0 and #{shard_total - 1}" unless + (0...shard_total).cover?(shard_index) + + @assigned_dependent_formula_names = nil + @handled_dependent_formula_names.clear + return if shard_total == 1 + + all_dependent_formula_names = @dependent_testing_formulae.flat_map do |formula_name| + formula = Formulary.factory(formula_name) + skip_recursive_dependents = skip_recursive_dependents_for(formula, args:) + dependent_formula_names(formula_name, skip_recursive_dependents:) + end.uniq.sort + # TODO: Consider sharding strategies that reduce duplicated dependency installs across shards. + @assigned_dependent_formula_names = Set.new( + all_dependent_formula_names.each_with_index.filter_map do |name, position| + name if position % shard_total == shard_index + end, + ) + end + + sig { params(dependents: T::Array[Formula]).returns(T::Array[Formula]) } + def sharded_dependents(dependents) + assigned_dependent_formula_names = @assigned_dependent_formula_names + return dependents if assigned_dependent_formula_names.nil? + + dependents.select do |dependent| + assigned_dependent_formula_names.include?(dependent.full_name) && + @handled_dependent_formula_names.exclude?(dependent.full_name) + end + end + sig { params( dependent: Formula,