From 0f6a9a4e9da7f29d7f23516fe1c8faa6adec9406 Mon Sep 17 00:00:00 2001 From: pedroSG94 Date: Wed, 5 Aug 2026 16:17:58 +0200 Subject: [PATCH 1/2] quantize video ts --- .../com/pedro/common/TimestampQuantizer.kt | 103 ++++++++++++++++++ .../pedro/common/TimestampQuantizerTest.kt | 100 +++++++++++++++++ .../com/pedro/encoder/video/VideoEncoder.java | 16 ++- 3 files changed, 215 insertions(+), 4 deletions(-) create mode 100644 common/src/main/java/com/pedro/common/TimestampQuantizer.kt create mode 100644 common/src/test/java/com/pedro/common/TimestampQuantizerTest.kt diff --git a/common/src/main/java/com/pedro/common/TimestampQuantizer.kt b/common/src/main/java/com/pedro/common/TimestampQuantizer.kt new file mode 100644 index 000000000..51111dcf3 --- /dev/null +++ b/common/src/main/java/com/pedro/common/TimestampQuantizer.kt @@ -0,0 +1,103 @@ +/* + * Copyright (C) 2026 pedroSG94. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.pedro.common + +import kotlin.math.abs +import kotlin.math.max +import kotlin.math.roundToLong + +/** + * Created by pedro on 4/8/26. + * + * Timestamps generated from the clock carry the jitter of the thread that reads it + * (in a render thread it can be +-15ms at 30fps). That jitter is harmless for a live + * player but it makes the result a variable frame rate stream, so ffprobe/players can't + * detect the frame rate of a recorded file and the playback looks jumpy. + * + * This snaps each timestamp to the closest slot of the expected frame rate grid, keeping + * the values close to the original clock, so the frame rate is stable without desync. + * + * Slots can be skipped (a 15fps source in a 30fps grid is preserved) but the value is never + * repeated or decreased. + * + * If the source isn't running at a rate that fits the grid the timestamp is returned + * untouched. This is the case of a screen source without changes on screen or a video file + * recorded in a different frame rate, where a grid would be a lie. + * + * All the values are in microseconds. + */ +class TimestampQuantizer { + + private var lastTimestamp = NO_VALUE + private var lastResult = NO_VALUE + private var anchorTimestamp = 0L + private var anchorResult = 0L + private var lastSlot = 0L + private var averageInterval = 0.0 + + fun reset() { + lastTimestamp = NO_VALUE + lastResult = NO_VALUE + averageInterval = 0.0 + } + + fun quantize(timestamp: Long, fps: Int): Long { + if (fps <= 0) return timestamp + val interval = 1_000_000.0 / fps + if (lastTimestamp == NO_VALUE) return anchor(timestamp, timestamp) + val rawInterval = (timestamp - lastTimestamp).toDouble() + lastTimestamp = timestamp + averageInterval = + if (averageInterval == 0.0) rawInterval + else averageInterval + (rawInterval - averageInterval) / SMOOTH_FACTOR + //the source can produce a frame per slot or skip slots (15fps in a 30fps grid) but a + //rate that doesn't fit the grid (24fps in a 30fps grid) would be worse quantized + val ratio = averageInterval / interval + val slotsPerFrame = ratio.roundToLong() + if (slotsPerFrame < 1 || abs(ratio - slotsPerFrame) > TOLERANCE) { + //the source isn't running at the expected fps, use the clock as is + return anchor(timestamp, max(lastResult + 1, timestamp)) + } + var slot = ((timestamp - anchorTimestamp) / interval).roundToLong() + if (slot <= lastSlot) slot = lastSlot + 1 + val result = anchorResult + (slot * interval).toLong() + //too far from the clock (frames faster than the grid), sync again to avoid desync + if (abs(result - timestamp) > interval) { + return anchor(timestamp, max(lastResult + interval.toLong(), timestamp)) + } + lastSlot = slot + lastResult = result + return result + } + + private fun anchor(timestamp: Long, result: Long): Long { + lastTimestamp = timestamp + anchorTimestamp = timestamp + anchorResult = result + lastSlot = 0 + lastResult = result + return result + } + + companion object { + private const val NO_VALUE = Long.MIN_VALUE + //number of frames used to calculate the real interval of the source + private const val SMOOTH_FACTOR = 8 + //max difference allowed between the real interval and the expected one + private const val TOLERANCE = 0.2 + } +} diff --git a/common/src/test/java/com/pedro/common/TimestampQuantizerTest.kt b/common/src/test/java/com/pedro/common/TimestampQuantizerTest.kt new file mode 100644 index 000000000..c9e27bf49 --- /dev/null +++ b/common/src/test/java/com/pedro/common/TimestampQuantizerTest.kt @@ -0,0 +1,100 @@ +package com.pedro.common + +import junit.framework.TestCase.assertEquals +import junit.framework.TestCase.assertTrue +import org.junit.Test + +class TimestampQuantizerTest { + + @Test + fun testJitterRemoved() { + val quantizer = TimestampQuantizer() + //clock timestamps of a 30fps source with the jitter of a render thread + val timestamps = listOf( + 0L, 30000L, 71000L, 95000L, 135000L, 160000L, 202000L, 229000L, + 270000L, 296000L, 335000L, 364000L, 399000L, 431000L, 468000L + ) + val result = timestamps.map { quantizer.quantize(it, 30) } + assertEquals( + listOf( + 0L, 33333L, 66666L, 100000L, 133333L, 166666L, 200000L, 233333L, + 266666L, 300000L, 333333L, 366666L, 400000L, 433333L, 466666L + ), + result + ) + } + + @Test + fun testSlowerSourceSkipSlots() { + val quantizer = TimestampQuantizer() + //source producing a frame every 2 slots (15fps in a 30fps grid) + val timestamps = listOf(0L, 68000L, 130000L, 202000L, 264000L) + val result = timestamps.map { quantizer.quantize(it, 30) } + assertEquals(listOf(0L, 66666L, 133333L, 200000L, 266666L), result) + } + + @Test + fun testRateThatDoesNotFitGridNotModified() { + val quantizer = TimestampQuantizer() + //24fps source in a 30fps grid, a grid would produce a worse result + val timestamps = listOf(0L, 41000L, 84000L, 125000L, 166000L, 209000L) + val result = timestamps.map { quantizer.quantize(it, 30) } + assertEquals(timestamps, result) + } + + @Test + fun testIrregularSourceNotModified() { + val quantizer = TimestampQuantizer() + //a source that isn't running at the expected fps is returned as is + val timestamps = listOf(0L, 500000L, 1000000L, 1500000L, 2000000L, 2500000L) + val result = timestamps.map { quantizer.quantize(it, 30) } + assertEquals(timestamps, result) + } + + @Test + fun testAlwaysIncrease() { + val quantizer = TimestampQuantizer() + //two frames rendered in the same clock value must not produce the same timestamp + val timestamps = listOf(0L, 33000L, 33000L, 66000L, 100000L, 133000L) + val result = timestamps.map { quantizer.quantize(it, 30) } + var last = -1L + result.forEach { + assertTrue("$it is not higher than $last", it > last) + last = it + } + } + + @Test + fun testNoDesyncWithClock() { + val quantizer = TimestampQuantizer() + //10 seconds of a 30fps source, the result must follow the clock closely + var timestamp = 0L + val jitter = listOf(-12000L, 5000L, 9000L, -7000L, 0L, 14000L, -3000L) + var maxDifference = 0L + repeat(300) { + val clock = timestamp + jitter[it % jitter.size] + val result = quantizer.quantize(clock, 30) + val difference = kotlin.math.abs(result - clock) + if (difference > maxDifference) maxDifference = difference + timestamp += 33333L + } + assertTrue("max difference with the clock: $maxDifference", maxDifference <= 33333L) + } + + @Test + fun testResetStartAgain() { + val quantizer = TimestampQuantizer() + listOf(0L, 33000L, 66000L, 100000L).forEach { quantizer.quantize(it, 30) } + quantizer.reset() + assertEquals(0L, quantizer.quantize(0L, 30)) + assertEquals(33333L, quantizer.quantize(34000L, 30)) + } + + @Test + fun testInvalidFpsNotModified() { + val quantizer = TimestampQuantizer() + val timestamps = listOf(0L, 30000L, 71000L, 95000L) + val result = timestamps.map { quantizer.quantize(it, 0) } + assertEquals(timestamps, result) + } +} diff --git a/encoder/src/main/java/com/pedro/encoder/video/VideoEncoder.java b/encoder/src/main/java/com/pedro/encoder/video/VideoEncoder.java index d5d0843e4..5f0440806 100644 --- a/encoder/src/main/java/com/pedro/encoder/video/VideoEncoder.java +++ b/encoder/src/main/java/com/pedro/encoder/video/VideoEncoder.java @@ -30,6 +30,7 @@ import androidx.annotation.RequiresApi; import com.pedro.common.TimeUtils; +import com.pedro.common.TimestampQuantizer; import com.pedro.common.VideoCodec; import com.pedro.encoder.BaseEncoder; import com.pedro.encoder.Frame; @@ -67,6 +68,7 @@ public class VideoEncoder extends BaseEncoder implements GetCameraData { private int rotation = 90; private int iFrameInterval = 2; private long firstTimestamp = 0; + private final TimestampQuantizer timestampQuantizer = new TimestampQuantizer(); //for disable video private final FpsLimiter fpsLimiter = new FpsLimiter(); private FormatVideoEncoder formatVideoEncoder = FormatVideoEncoder.YUV420Dynamical; @@ -197,7 +199,10 @@ public boolean prepareVideoEncoder(int width, int height, int fps, int bitRate, @Override public void start(boolean resetTs) { - if (resetTs) firstTimestamp = 0; + if (resetTs) { + firstTimestamp = 0; + timestampQuantizer.reset(); + } forceKey = false; spsPpsSetted = false; if (formatVideoEncoder != FormatVideoEncoder.SURFACE) { @@ -506,12 +511,15 @@ protected boolean checkBuffer(@NonNull ByteBuffer byteBuffer, @NonNull MediaCode // Buffer mode: synthesize PTS from wall clock. bufferInfo.presentationTimeUs = TimeUtils.getCurrentTimeMicro() - presentTimeUs; } else { - // Surface mode: EGL timestamp is camera sensor time (nanoseconds from boot รท 1000). - // It has clean, jitter-free intervals โ€” but it's a huge absolute value that breaks RTMP. - // Rebase to relative by subtracting the first frame's PTS โ†’ clean intervals, starts at 0. + // Surface mode: the EGL timestamp is the clock read by the render thread + // (GlInterface#setPresentationTime), a huge absolute value that breaks RTMP. + // Rebase to relative by subtracting the first frame's PTS โ†’ starts at 0. if (firstTimestamp == 0) firstTimestamp = bufferInfo.presentationTimeUs; bufferInfo.presentationTimeUs -= firstTimestamp; } + // Both cases come from the clock, so they carry the jitter of the thread that read it. + // Remove it to produce a constant frame rate that players can detect. + bufferInfo.presentationTimeUs = timestampQuantizer.quantize(bufferInfo.presentationTimeUs, fps); } else { if (firstTimestamp == 0) firstTimestamp = bufferInfo.presentationTimeUs; bufferInfo.presentationTimeUs -= firstTimestamp; From 225ca9db69677ecf6649ba9917054bffa6d2cc20 Mon Sep 17 00:00:00 2001 From: pedroSG94 Date: Wed, 5 Aug 2026 17:37:18 +0200 Subject: [PATCH 2/2] fix quantizer when real fps is higher than fps target --- .../com/pedro/common/TimestampQuantizer.kt | 62 +++++++++++++++---- .../pedro/common/TimestampQuantizerTest.kt | 27 +++++++- 2 files changed, 77 insertions(+), 12 deletions(-) diff --git a/common/src/main/java/com/pedro/common/TimestampQuantizer.kt b/common/src/main/java/com/pedro/common/TimestampQuantizer.kt index 51111dcf3..0e2571634 100644 --- a/common/src/main/java/com/pedro/common/TimestampQuantizer.kt +++ b/common/src/main/java/com/pedro/common/TimestampQuantizer.kt @@ -34,7 +34,12 @@ import kotlin.math.roundToLong * Slots can be skipped (a 15fps source in a 30fps grid is preserved) but the value is never * repeated or decreased. * - * If the source isn't running at a rate that fits the grid the timestamp is returned + * A source faster than the configured fps (a 60fps video file streamed at 30fps) is snapped + * to half the grid instead. That case is detected checking that the frames really land on the + * half grid, the average interval isn't enough because those sources skip slots in a + * irregular way. + * + * If the source isn't running at a rate that fits any of both grids the timestamp is returned * untouched. This is the case of a screen source without changes on screen or a video file * recorded in a different frame rate, where a grid would be a lie. * @@ -48,48 +53,79 @@ class TimestampQuantizer { private var anchorResult = 0L private var lastSlot = 0L private var averageInterval = 0.0 + private var gridError = 0.0 + private var halfGridError = 0.0 + private var gridInterval = 0.0 + private var frames = 0 + private var halfVotes = 0 fun reset() { lastTimestamp = NO_VALUE lastResult = NO_VALUE averageInterval = 0.0 + gridError = 0.0 + halfGridError = 0.0 + gridInterval = 0.0 + frames = 0 + halfVotes = 0 } fun quantize(timestamp: Long, fps: Int): Long { if (fps <= 0) return timestamp - val interval = 1_000_000.0 / fps - if (lastTimestamp == NO_VALUE) return anchor(timestamp, timestamp) + val nominal = 1_000_000.0 / fps + if (lastTimestamp == NO_VALUE) return anchor(timestamp, timestamp, nominal) val rawInterval = (timestamp - lastTimestamp).toDouble() lastTimestamp = timestamp + frames++ averageInterval = if (averageInterval == 0.0) rawInterval else averageInterval + (rawInterval - averageInterval) / SMOOTH_FACTOR - //the source can produce a frame per slot or skip slots (15fps in a 30fps grid) but a - //rate that doesn't fit the grid (24fps in a 30fps grid) would be worse quantized - val ratio = averageInterval / interval + val half = nominal / 2 + gridError = fitError(gridError, rawInterval / nominal) + halfGridError = fitError(halfGridError, rawInterval / half) + + //faster than the grid and landing on the half grid (60fps file in a 30fps stream). + //The half grid has denser slots so it can fit a jittery source by chance: only use it if + //it fits clearly better than the nominal one and it does it in a sustained way + val fitsHalf = frames > SMOOTH_FACTOR && averageInterval >= half * MIN_FILL + && halfGridError <= TOLERANCE && halfGridError < gridError * BETTER_FIT + halfVotes = if (fitsHalf) halfVotes + 1 else 0 + + val ratio = averageInterval / nominal val slotsPerFrame = ratio.roundToLong() - if (slotsPerFrame < 1 || abs(ratio - slotsPerFrame) > TOLERANCE) { - //the source isn't running at the expected fps, use the clock as is - return anchor(timestamp, max(lastResult + 1, timestamp)) + val interval = when { + //a frame per slot or skipping slots in a regular way (15fps in a 30fps grid) + slotsPerFrame >= 1 && abs(ratio - slotsPerFrame) <= TOLERANCE -> nominal + halfVotes >= SMOOTH_FACTOR -> half + //the rate doesn't fit any grid, using a grid would be a lie + else -> return anchor(timestamp, max(lastResult + 1, timestamp), nominal) } + if (interval != gridInterval) return anchor(timestamp, max(lastResult + 1, timestamp), interval) var slot = ((timestamp - anchorTimestamp) / interval).roundToLong() if (slot <= lastSlot) slot = lastSlot + 1 val result = anchorResult + (slot * interval).toLong() //too far from the clock (frames faster than the grid), sync again to avoid desync if (abs(result - timestamp) > interval) { - return anchor(timestamp, max(lastResult + interval.toLong(), timestamp)) + return anchor(timestamp, max(lastResult + interval.toLong(), timestamp), interval) } lastSlot = slot lastResult = result return result } - private fun anchor(timestamp: Long, result: Long): Long { + //how far the interval is from landing on a grid slot, 0 = exactly on a slot + private fun fitError(current: Double, slots: Double): Double { + val error = abs(slots - slots.roundToLong()) + return if (current == 0.0) error else current + (error - current) / SMOOTH_FACTOR + } + + private fun anchor(timestamp: Long, result: Long, interval: Double): Long { lastTimestamp = timestamp anchorTimestamp = timestamp anchorResult = result lastSlot = 0 lastResult = result + gridInterval = interval return result } @@ -99,5 +135,9 @@ class TimestampQuantizer { private const val SMOOTH_FACTOR = 8 //max difference allowed between the real interval and the expected one private const val TOLERANCE = 0.2 + //the half grid can't be used if the source is slower than it + private const val MIN_FILL = 0.8 + //how much better the half grid must fit to be used instead of the nominal one + private const val BETTER_FIT = 0.75 } } diff --git a/common/src/test/java/com/pedro/common/TimestampQuantizerTest.kt b/common/src/test/java/com/pedro/common/TimestampQuantizerTest.kt index c9e27bf49..7ef44f0a2 100644 --- a/common/src/test/java/com/pedro/common/TimestampQuantizerTest.kt +++ b/common/src/test/java/com/pedro/common/TimestampQuantizerTest.kt @@ -46,11 +46,36 @@ class TimestampQuantizerTest { fun testIrregularSourceNotModified() { val quantizer = TimestampQuantizer() //a source that isn't running at the expected fps is returned as is - val timestamps = listOf(0L, 500000L, 1000000L, 1500000L, 2000000L, 2500000L) + val timestamps = listOf(0L, 45000L, 100000L, 145000L, 200000L, 245000L) val result = timestamps.map { quantizer.quantize(it, 30) } assertEquals(timestamps, result) } + @Test + fun testFasterSourceUsesHalfGrid() { + val quantizer = TimestampQuantizer() + //60fps source in a 30fps grid dropping frames in a irregular way, like a video file + val timestamps = listOf( + 0L, 17000L, 33000L, 51000L, 66000L, 100000L, 116000L, 133000L, 151000L, + 166000L, 200000L, 216000L, 233000L, 249000L, 267000L, 283000L, 300000L, + 317000L, 333000L, 351000L, 366000L, 400000L, 416000L, 433000L, 451000L, + 466000L, 483000L, 500000L, 517000L, 533000L + ) + val result = timestamps.map { quantizer.quantize(it, 30) } + val deltas = result.zipWithNext { a, b -> b - a } + //passes through until the rate is confirmed, then it snaps to the half grid (16666us) + //skipping a slot when the source drops a frame + assertEquals( + listOf( + 17000L, 16000L, 18000L, 15000L, 34000L, 16000L, 17000L, 18000L, 15000L, + 34000L, 16000L, 17000L, 16000L, 18000L, 16000L, 17000L, 16666L, 16667L, + 16667L, 16666L, 33334L, 16666L, 16667L, 16667L, 16666L, 16667L, 16667L, + 16666L, 16667L + ), + deltas + ) + } + @Test fun testAlwaysIncrease() { val quantizer = TimestampQuantizer()