Skip to content

Commit 1c6cfd4

Browse files
jimm98yjimm98y
authored
Merge pull request #12 from jimm98y/features/jpg-sample
F Added JPG sample, image width/height is now parsed from JPEG
2 parents 61d93b8 + afb9369 commit 1c6cfd4

6 files changed

Lines changed: 165 additions & 93 deletions

File tree

‎src/RTSPServerApp/Images/image001.jpg‎

235 KB
Loading

‎src/RTSPServerApp/Images/image002.jpg‎

233 KB
Loading

‎src/RTSPServerApp/Images/image003.jpg‎

233 KB
Loading

‎src/RTSPServerApp/RTSPServerApp.csproj‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,31 @@
77
<Nullable>disable</Nullable>
88
</PropertyGroup>
99

10+
<ItemGroup>
11+
<None Remove="Images\image001.jpg" />
12+
<None Remove="Images\image002.jpg" />
13+
<None Remove="Images\image003.jpg" />
14+
<None Remove="Images\sample-birch-400x300.jpg" />
15+
<None Remove="Images\sample-blue-400x300.jpg" />
16+
<None Remove="Images\sample-city-park-400x300.jpg" />
17+
<None Remove="Images\sample-clouds-400x300.jpg" />
18+
<None Remove="Images\sample-green-400x300.jpg" />
19+
<None Remove="Images\sample-red-400x300.jpg" />
20+
</ItemGroup>
21+
1022
<ItemGroup>
1123
<Content Include="frag_bunny.mp4">
1224
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
1325
</Content>
26+
<Content Include="Images\image001.jpg">
27+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
28+
</Content>
29+
<Content Include="Images\image002.jpg">
30+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
31+
</Content>
32+
<Content Include="Images\image003.jpg">
33+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
34+
</Content>
1435
</ItemGroup>
1536

1637
<ItemGroup>

‎src/RTSPServerApp/RTSPServerWorker.cs‎

Lines changed: 62 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -50,69 +50,89 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
5050
ITrack rtspVideoTrack = null;
5151
ITrack rtspAudioTrack = null;
5252

53-
// frag_bunny.mp4 audio is not playable in VLC on Windows 11 (works on MacOS)
54-
using (Stream fs = new BufferedStream(new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.Read)))
53+
if (Path.GetExtension(fileName) == ".mp4")
5554
{
56-
using (var fmp4 = await FragmentedMp4.ParseAsync(fs))
55+
// frag_bunny.mp4 audio is not playable in VLC on Windows 11 (works on MacOS)
56+
using (Stream fs = new BufferedStream(new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.Read)))
5757
{
58-
videoTrackBox = fmp4.FindVideoTracks().FirstOrDefault();
59-
audioTrackBox = fmp4.FindAudioTracks().FirstOrDefault();
60-
61-
parsedMDAT = await fmp4.ParseMdatAsync();
62-
63-
if (videoTrackBox != null)
58+
using (var fmp4 = await FragmentedMp4.ParseAsync(fs))
6459
{
65-
videoTrackId = fmp4.FindVideoTrackID().First();
66-
videoFrameRate = fmp4.CalculateFrameRate(videoTrackBox);
60+
videoTrackBox = fmp4.FindVideoTracks().FirstOrDefault();
61+
audioTrackBox = fmp4.FindAudioTracks().FirstOrDefault();
6762

68-
var h264VisualSample = videoTrackBox.GetMdia().GetMinf().GetStbl().GetStsd().Children.FirstOrDefault(x => x.Type == VisualSampleEntryBox.TYPE3 || x.Type == VisualSampleEntryBox.TYPE4) as VisualSampleEntryBox;
69-
if (h264VisualSample != null)
70-
{
71-
var avcC = (h264VisualSample.Children.First(x => x.Type == AvcConfigurationBox.TYPE) as AvcConfigurationBox).AvcDecoderConfigurationRecord;
72-
rtspVideoTrack = new SharpRTSPServer.H264Track(avcC.AvcProfileIndication, 0, avcC.AvcLevelIndication);
73-
_server.AddVideoTrack(rtspVideoTrack);
74-
}
75-
else
63+
parsedMDAT = await fmp4.ParseMdatAsync();
64+
65+
if (videoTrackBox != null)
7666
{
77-
var h265VisualSample = videoTrackBox.GetMdia().GetMinf().GetStbl().GetStsd().Children.FirstOrDefault(x => x.Type == VisualSampleEntryBox.TYPE6 || x.Type == VisualSampleEntryBox.TYPE7) as VisualSampleEntryBox;
78-
if (h265VisualSample != null)
67+
videoTrackId = fmp4.FindVideoTrackID().First();
68+
videoFrameRate = fmp4.CalculateFrameRate(videoTrackBox);
69+
70+
var h264VisualSample = videoTrackBox.GetMdia().GetMinf().GetStbl().GetStsd().Children.FirstOrDefault(x => x.Type == VisualSampleEntryBox.TYPE3 || x.Type == VisualSampleEntryBox.TYPE4) as VisualSampleEntryBox;
71+
if (h264VisualSample != null)
7972
{
80-
rtspVideoTrack = new SharpRTSPServer.H265Track();
73+
var avcC = (h264VisualSample.Children.First(x => x.Type == AvcConfigurationBox.TYPE) as AvcConfigurationBox).AvcDecoderConfigurationRecord;
74+
rtspVideoTrack = new SharpRTSPServer.H264Track(avcC.AvcProfileIndication, 0, avcC.AvcLevelIndication);
8175
_server.AddVideoTrack(rtspVideoTrack);
8276
}
8377
else
8478
{
85-
throw new NotSupportedException("No supported video found!");
79+
var h265VisualSample = videoTrackBox.GetMdia().GetMinf().GetStbl().GetStsd().Children.FirstOrDefault(x => x.Type == VisualSampleEntryBox.TYPE6 || x.Type == VisualSampleEntryBox.TYPE7) as VisualSampleEntryBox;
80+
if (h265VisualSample != null)
81+
{
82+
rtspVideoTrack = new SharpRTSPServer.H265Track();
83+
_server.AddVideoTrack(rtspVideoTrack);
84+
}
85+
else
86+
{
87+
throw new NotSupportedException("No supported video found!");
88+
}
8689
}
8790
}
88-
}
89-
90-
if (audioTrackBox != null)
91-
{
92-
audioTrackId = fmp4.FindAudioTrackID().First();
9391

94-
var audioSampleEntry = audioTrackBox.GetAudioSampleEntryBox();
95-
if (audioSampleEntry.Type == AudioSampleEntryBox.TYPE3) // AAC
92+
if (audioTrackBox != null)
9693
{
97-
var audioConfigDescriptor = audioSampleEntry.GetAudioSpecificConfigDescriptor();
98-
int audioSamplingRate = audioConfigDescriptor.GetSamplingFrequency();
99-
rtspAudioTrack = new SharpRTSPServer.AACTrack(await audioConfigDescriptor.ToBytes(), audioSamplingRate, audioConfigDescriptor.ChannelConfiguration);
100-
_server.AddAudioTrack(rtspAudioTrack);
101-
}
102-
else
103-
{
104-
// unsupported audio
94+
audioTrackId = fmp4.FindAudioTrackID().First();
95+
96+
var audioSampleEntry = audioTrackBox.GetAudioSampleEntryBox();
97+
if (audioSampleEntry.Type == AudioSampleEntryBox.TYPE3) // AAC
98+
{
99+
var audioConfigDescriptor = audioSampleEntry.GetAudioSpecificConfigDescriptor();
100+
int audioSamplingRate = audioConfigDescriptor.GetSamplingFrequency();
101+
rtspAudioTrack = new SharpRTSPServer.AACTrack(await audioConfigDescriptor.ToBytes(), audioSamplingRate, audioConfigDescriptor.ChannelConfiguration);
102+
_server.AddAudioTrack(rtspAudioTrack);
103+
}
104+
else
105+
{
106+
// unsupported audio
107+
}
105108
}
106109
}
107110
}
108111
}
112+
else
113+
{
114+
parsedMDAT = new Dictionary<uint, IList<IList<byte[]>>>();
115+
parsedMDAT.Add(videoTrackId, new List<IList<byte[]>>());
116+
117+
var jpgFiles = Directory.GetFiles(fileName, "*.jpg");
118+
for (int i = 0; i < jpgFiles.Length; i++)
119+
{
120+
parsedMDAT[videoTrackId].Add(new List<byte[]>());
121+
parsedMDAT[videoTrackId][i].Add(File.ReadAllBytes(jpgFiles[i]));
122+
}
123+
124+
rtspVideoTrack = new SharpRTSPServer.MJpegTrack();
125+
_server.AddVideoTrack(rtspVideoTrack);
126+
127+
videoFrameRate = 1;
128+
}
109129

110130
int videoIndex = 0;
111131
int audioIndex = 0;
112132
Timer audioTimer = null;
113133
Timer videoTimer = null;
114134

115-
if (videoTrackBox != null)
135+
if (rtspVideoTrack != null)
116136
{
117137
var videoSamplingRate = SharpRTSPServer.H264Track.DEFAULT_CLOCK;
118138
var videoSampleDuration = videoSamplingRate / videoFrameRate;
@@ -125,12 +145,13 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
125145
if (rtspVideoTrack is SharpRTSPServer.H264Track h264VideoTrack)
126146
{
127147
h264VideoTrack.SetParameterSets(videoTrack[0][0], videoTrack[0][1]);
148+
videoIndex++;
128149
}
129150
else if (rtspVideoTrack is SharpRTSPServer.H265Track h265VideoTrack)
130151
{
131152
h265VideoTrack.SetParameterSets(videoTrack[0][0], videoTrack[0][1], videoTrack[0][2]);
153+
videoIndex++;
132154
}
133-
videoIndex++;
134155
}
135156

136157
rtspVideoTrack.FeedInRawSamples((uint)(videoIndex * videoSampleDuration), (List<byte[]>)videoTrack[videoIndex++ % videoTrack.Count]);
@@ -142,7 +163,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
142163
};
143164
}
144165

145-
if (audioTrackBox != null)
166+
if (rtspAudioTrack != null)
146167
{
147168
var audioSampleDuration = SharpMp4.AACTrack.AAC_SAMPLE_SIZE;
148169
var audioTrack = parsedMDAT[audioTrackId];

‎src/SharpRTSPServer/MJpegTrack.cs‎

Lines changed: 82 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,6 @@ public class MJpegTrack : TrackBase
2121
/// <inheritdoc/>
2222
public override int ID { get; set; }
2323

24-
public int Width { get; set; }
25-
26-
public int Height { get; set; }
27-
2824
/// <inheritdoc/>
2925
public override StringBuilder BuildSDP(StringBuilder sdp)
3026
{
@@ -51,8 +47,6 @@ public override (List<Memory<byte>>, List<IMemoryOwner<byte>>) CreateRtpPackets(
5147
const ushort SoiMarker = 0xFFD8; // SOI - Start of image header
5248
//const ushort App0Header = 0xFFE0; // Application Segment 0 header
5349
//const ushort App15Header = 0xFFEF; // Application Segment 15 header
54-
const ushort Sof0Marker = 0xFFC0; // SOF0 - Start of Frame marker
55-
const ushort DqtHeader = 0xFFDB; // Define Quantization Table header
5650
//const ushort SosMarker = 0xFFDA; // SOS - Start of Scan marker
5751
const ushort EoiMarker = 0xFFD9; // EOI - End of Image marker
5852

@@ -70,52 +64,14 @@ public override (List<Memory<byte>>, List<IMemoryOwner<byte>>) CreateRtpPackets(
7064
throw new InvalidOperationException($"JPEG image must start with SOI marker {EoiMarker.ToString("X4")} and {header.ToString("X4")}");
7165
}
7266

73-
//var reader = jpegImage[2..^2];
74-
var reader = jpegImage.Slice(2); // keep EOI
75-
7667
byte type = 1; // https://datatracker.ietf.org/doc/html/rfc2435#section-3.1.3
7768
byte q = 255; // https://datatracker.ietf.org/doc/html/rfc2435#section-3.1.4, https://datatracker.ietf.org/doc/html/rfc2435#section-4.2
7869

79-
int nbQuantizationTables = 0;
8070
var firstQuantizationtable = ReadOnlySpan<byte>.Empty;
8171
var secondQuantizationtable = ReadOnlySpan<byte>.Empty;
8272

83-
while (true)
84-
{
85-
header = BinaryPrimitives.ReadUInt16BigEndian(reader);
86-
87-
if (header == Sof0Marker)
88-
{
89-
break;
90-
}
91-
92-
reader = reader.Slice(2);
93-
94-
var size = BinaryPrimitives.ReadUInt16BigEndian(reader) - 2; reader = reader.Slice(2);
95-
96-
switch (header)
97-
{
98-
case DqtHeader:
99-
nbQuantizationTables++;
100-
if (nbQuantizationTables == 1)
101-
{
102-
firstQuantizationtable = reader.Slice(0, size);
103-
}
104-
else if (nbQuantizationTables == 2)
105-
{
106-
secondQuantizationtable = reader.Slice(0, size);
107-
}
108-
else
109-
{
110-
throw new InvalidOperationException("Error: More than 2 quantization tables in JPEG image");
111-
}
112-
break;
113-
default:
114-
break;
115-
}
116-
117-
reader = reader.Slice(size);
118-
}
73+
Span<byte> reader;
74+
var jpegSize = ParseJpeg(jpegImage, out firstQuantizationtable, out secondQuantizationtable, out reader);
11975

12076
// Build a list of 1 or more RTP packets
12177
// The last packet will have the M bit set to '1'
@@ -178,8 +134,8 @@ public override (List<Memory<byte>>, List<IMemoryOwner<byte>>) CreateRtpPackets(
178134
// Write JPEG Header - https://datatracker.ietf.org/doc/html/rfc2435#section-3.1
179135
rtpPacketSpan[0] = type;
180136
rtpPacketSpan[1] = q;
181-
rtpPacketSpan[2] = (byte)(Width / 8);
182-
rtpPacketSpan[3] = (byte)(Height / 8);
137+
rtpPacketSpan[2] = (byte)(jpegSize.width >> 3);
138+
rtpPacketSpan[3] = (byte)(jpegSize.height >> 3);
183139
rtpPacketSpan = rtpPacketSpan.Slice(4);
184140

185141
// write quantization tables
@@ -190,7 +146,7 @@ public override (List<Memory<byte>>, List<IMemoryOwner<byte>>) CreateRtpPackets(
190146

191147
// Write Quantization Table header https://datatracker.ietf.org/doc/html/rfc2435#section-3.1.8
192148

193-
if (nbQuantizationTables == 1)
149+
if (secondQuantizationtable.IsEmpty)
194150
{
195151
// MBZ
196152
rtpPacketSpan[0] = (byte)(firstQuantizationtable[0] & 0xf);
@@ -200,7 +156,7 @@ public override (List<Memory<byte>>, List<IMemoryOwner<byte>>) CreateRtpPackets(
200156

201157
// Length
202158
var qtSize = firstQuantizationtable.Length - 1;
203-
BinaryPrimitives.WriteInt16BigEndian(rtpPacketSpan.Slice(2), (short)(qtSize));
159+
BinaryPrimitives.WriteInt16BigEndian(rtpPacketSpan.Slice(2), (short)qtSize);
204160

205161
// Quantization Table Data
206162
firstQuantizationtable.Slice(1).CopyTo(rtpPacketSpan.Slice(4));
@@ -218,7 +174,7 @@ public override (List<Memory<byte>>, List<IMemoryOwner<byte>>) CreateRtpPackets(
218174

219175
// Length
220176
var qtSize = firstQuantizationtable.Length + secondQuantizationtable.Length - 2;
221-
BinaryPrimitives.WriteInt16BigEndian(rtpPacketSpan.Slice(2), (short)(qtSize));
177+
BinaryPrimitives.WriteInt16BigEndian(rtpPacketSpan.Slice(2), (short)qtSize);
222178

223179
// Quantization Table Data
224180
firstQuantizationtable.Slice(1).CopyTo(rtpPacketSpan.Slice(4));
@@ -230,7 +186,6 @@ public override (List<Memory<byte>>, List<IMemoryOwner<byte>>) CreateRtpPackets(
230186
}
231187

232188
// Write JPEG Payload
233-
234189
reader.Slice(0, rtpPacketSpan.Length).CopyTo(rtpPacketSpan);
235190
reader = reader.Slice(rtpPacketSpan.Length);
236191
dataPointer += rtpPacketSpan.Length;
@@ -241,5 +196,80 @@ public override (List<Memory<byte>>, List<IMemoryOwner<byte>>) CreateRtpPackets(
241196

242197
return (rtpPackets, memoryOwners);
243198
}
199+
200+
201+
private static (int width, int height, int bpp) ParseJpeg(Span<byte> binaryReader, out ReadOnlySpan<byte> first, out ReadOnlySpan<byte> second, out Span<byte> retReader)
202+
{
203+
first = ReadOnlySpan<byte>.Empty;
204+
second = ReadOnlySpan<byte>.Empty;
205+
Span<byte> br = binaryReader;
206+
207+
// JPG magic bytes
208+
if (br[0] != 0xff || br[1] != 0xd8)
209+
{
210+
throw new ArgumentException();
211+
}
212+
213+
br = br.Slice(2);
214+
215+
while (br[0] == 0xff)
216+
{
217+
// Start-Of-Frame (SOF) has 4 possible values
218+
if (br[1] == 0xc0 || br[1] == 0xc1 || br[1] == 0xc2 || br[1] == 0xc3)
219+
{
220+
retReader = br;
221+
222+
br = br.Slice(2);
223+
br = br.Slice(2);
224+
225+
// bits per pixel
226+
int bpp = br[0];
227+
br = br.Slice(1);
228+
229+
// image height
230+
int height = (br[0] << 8) | br[1];
231+
br = br.Slice(2);
232+
233+
// image width
234+
int width = (br[0] << 8) | br[1];
235+
br = br.Slice(2);
236+
237+
return (width, height, bpp);
238+
}
239+
240+
br = br.Slice(1);
241+
242+
byte marker = br[0];
243+
br = br.Slice(1);
244+
245+
short chunkLength = (short)((br[0] << 8) | br[1]);
246+
br = br.Slice(2);
247+
248+
// quantization tables
249+
if (marker == 0xdb)
250+
{
251+
int matrix_length = chunkLength - 2;
252+
var matrix = br.Slice(0, matrix_length);
253+
if (first.IsEmpty)
254+
first = matrix;
255+
else if (second.IsEmpty)
256+
second = matrix;
257+
else
258+
throw new InvalidOperationException("Error: More than 2 quantization tables in JPEG image");
259+
}
260+
261+
if (chunkLength < 0)
262+
{
263+
ushort uchunkLength = (ushort)chunkLength;
264+
br = br.Slice(uchunkLength - 2);
265+
}
266+
else
267+
{
268+
br = br.Slice(chunkLength - 2);
269+
}
270+
}
271+
272+
throw new ArgumentException();
273+
}
244274
}
245275
}

0 commit comments

Comments
 (0)