hls fmp4: support audio only and video only;

Author: suzp1984

trunk/src/app/srs_app_hls.cpp

@@ -93,6 +93,53 @@ srs_error_t SrsInitMp4Segment::write(SrsFormat* format, int v_tid, int a_tid)
 {
     srs_error_t err = srs_success;
 
+    if ((err = init_encoder()) != srs_success) {
+        return srs_error_wrap(err, "init encoder");
+    }
+    
+    if ((err = init_->write(format, v_tid, a_tid)) != srs_success) {
+        return srs_error_wrap(err, "write init");
+    }
+    
+    return err;
+}
+
+srs_error_t SrsInitMp4Segment::write_video_only(SrsFormat* format, int v_tid)
+{
+    srs_error_t err = srs_success;
+        
+    if ((err = init_encoder()) != srs_success) {
+        return srs_error_wrap(err, "init encoder");
+    }
+    
+    if ((err = init_->write(format, true, v_tid)) != srs_success) {
+        return srs_error_wrap(err, "write init");
+    }
+    
+    return err;
+}
+
+srs_error_t SrsInitMp4Segment::write_audio_only(SrsFormat* format, int a_tid)
+{
+    srs_error_t err = srs_success;
+        
+    if ((err = init_encoder()) != srs_success) {
+        return srs_error_wrap(err, "init encoder");
+    }
+    
+    if ((err = init_->write(format, false, a_tid)) != srs_success) {
+        return srs_error_wrap(err, "write init");
+    }
+    
+    return err;
+}
+
+srs_error_t SrsInitMp4Segment::init_encoder()
+{
+    srs_error_t err = srs_success;
+
+    srs_assert(!fullpath().empty());
+
     string path_tmp = tmppath();
     if ((err = fw_->open(path_tmp)) != srs_success) {
         return srs_error_wrap(err, "Open init mp4 failed, path=%s", path_tmp.c_str());
@@ -101,14 +148,11 @@ srs_error_t SrsInitMp4Segment::write(SrsFormat* format, int v_tid, int a_tid)
     if ((err = init_->initialize(fw_)) != srs_success) {
         return srs_error_wrap(err, "init");
     }
-    
-    if ((err = init_->write(format, v_tid, a_tid)) != srs_success) {
-        return srs_error_wrap(err, "write init");
-    }
-    
+
     return err;
 }
 
+
 SrsHlsM4sSegment::SrsHlsM4sSegment()
 {
     fw_ = new SrsFileWriter();
@@ -428,7 +472,7 @@ srs_error_t SrsHlsFmp4Muxer::on_publish(SrsRequest* req)
     return err;
 }
 
-srs_error_t SrsHlsFmp4Muxer::write_init_mp4(SrsFormat* format)
+srs_error_t SrsHlsFmp4Muxer::write_init_mp4(SrsFormat* format, bool has_video, bool has_audio)
 {
     srs_error_t err = srs_success;
 
@@ -448,8 +492,20 @@ srs_error_t SrsHlsFmp4Muxer::write_init_mp4(SrsFormat* format)
 
     init_mp4->set_path(path);
 
-    if ((err = init_mp4->write(format, video_track_id_, audio_track_id_)) != srs_success) {
-        return srs_error_wrap(err, "write hls init.mp4 with audio and video");
+    if (has_video && has_audio) {
+        if ((err = init_mp4->write(format, video_track_id_, audio_track_id_)) != srs_success) {
+            return srs_error_wrap(err, "write hls init.mp4 with audio and video");
+        }
+    } else if (has_video) {
+        if ((err = init_mp4->write_video_only(format, video_track_id_)) != srs_success) {
+            return srs_error_wrap(err, "write hls init.mp4 with video only");
+        }
+    } else if (has_audio) {
+        if ((err = init_mp4->write_audio_only(format, audio_track_id_)) != srs_success) {
+            return srs_error_wrap(err, "write hls init.mp4 with audio only");
+        }
+    } else {
+        return srs_error_new(ERROR_HLS_WRITE_FAILED, "no video and no audio sequence header");
     }
 
     if ((err = init_mp4->rename()) != srs_success) {
@@ -471,6 +527,16 @@ srs_error_t SrsHlsFmp4Muxer::write_audio(SrsSharedPtrMessage* shared_audio, SrsF
             return srs_error_wrap(err, "open segment");
         }
     }
+
+    if (current_->duration() >= hls_fragment_) {
+        if ((err = segment_close()) != srs_success) {
+            return srs_error_wrap(err, "segment close");
+        }
+
+        if ((err = segment_open(shared_audio->timestamp * SRS_UTIME_MILLISECONDS)) != srs_success) {
+            return srs_error_wrap(err, "open segment");
+        }
+    }
 
     current_->write(shared_audio, format);
     return err;
@@ -488,7 +554,8 @@ srs_error_t SrsHlsFmp4Muxer::write_video(SrsSharedPtrMessage* shared_video, SrsF
         }
     }
 
-    bool reopen = format->video->frame_type == SrsVideoAvcFrameTypeKeyFrame && current_->duration() >= hls_fragment_;
+    // TODO: reap segment only when get key frame?
+    bool reopen = current_->duration() >= hls_fragment_;
     if (reopen) {
         if ((err = segment_close()) != srs_success) {
             return srs_error_wrap(err, "segment close");
@@ -499,7 +566,6 @@ srs_error_t SrsHlsFmp4Muxer::write_video(SrsSharedPtrMessage* shared_video, SrsF
         }
     }
 
-    // TODO: do reap segment here.
     current_->write(shared_video, format);
 
     return err;
@@ -2046,7 +2112,7 @@ srs_error_t SrsHlsMp4Controller::on_sequence_header(SrsSharedPtrMessage* msg, Sr
         has_audio_sh_ = true;
     }
 
-    muxer_->write_init_mp4(format);
+    muxer_->write_init_mp4(format, has_video_sh_, has_audio_sh_);
 
     return err;
 }
@@ -2291,8 +2357,14 @@ void SrsHls::on_unpublish()
 
 srs_error_t SrsHls::on_audio(SrsSharedPtrMessage* shared_audio, SrsFormat* format)
 {
+    // srs_warn("hls: on_audio, is_aac_sh=%s, aac_extra_data.size=%d", format->is_aac_sequence_header() ? "true" : "false", format->acodec->aac_extra_data.size());
+    
     srs_error_t err = srs_success;
 
+    if (format->acodec->aac_extra_data.size() == 0) {
+        srs_trace("the audio codec's aac extra data is empty");
+        return err;
+    }
     // If not able to transmux to HLS, ignore.
     if (!enabled || unpublishing_) return err;
     if (async_reload_) return reload();
@@ -2317,7 +2389,8 @@ srs_error_t SrsHls::on_audio(SrsSharedPtrMessage* shared_audio, SrsFormat* forma
 
     // ignore sequence header
     srs_assert(format->audio);
-    if (acodec == SrsAudioCodecIdAAC && format->audio->aac_packet_type == SrsAudioAacFrameTraitSequenceHeader) {
+    // TODO: verify mp3 play by HLS.
+    if (format->is_aac_sequence_header() || format->is_mp3_sequence_header()) {
         return controller->on_sequence_header(audio.get(), format);
     }
 
@@ -2335,6 +2408,7 @@ srs_error_t SrsHls::on_audio(SrsSharedPtrMessage* shared_audio, SrsFormat* forma
 
 srs_error_t SrsHls::on_video(SrsSharedPtrMessage* shared_video, SrsFormat* format)
 {
+    // srs_warn("hls: on_video, is_avc_sh=%s", format->is_avc_sequence_header() ? "true" : "false");
     srs_error_t err = srs_success;
 
     // If not able to transmux to HLS, ignore.
@@ -2364,8 +2438,10 @@ srs_error_t SrsHls::on_video(SrsSharedPtrMessage* shared_video, SrsFormat* forma
         return err;
     }
 
-    // ignore sequence header
-    if (format->video->avc_packet_type == SrsVideoAvcFrameTraitSequenceHeader) {
+    // ignore sequence header avc and hevc
+    // is avc|hevc|av1 sequence header check, but av1 packet already ignored above. so it's ok to use
+    // below method.
+    if (format->is_avc_sequence_header()) {
         return controller->on_sequence_header(video.get(), format);
     }
 

trunk/src/app/srs_app_hls.hpp

@@ -78,6 +78,12 @@ class SrsInitMp4Segment : public SrsFragment
 
     // Write the init mp4 file, with the v_tid(video track id) and a_tid (audio track id).
     virtual srs_error_t write(SrsFormat* format, int v_tid, int a_tid);
+
+    virtual srs_error_t write_video_only(SrsFormat* format, int v_tid);
+    virtual srs_error_t write_audio_only(SrsFormat* format, int a_tid);
+private:
+    virtual srs_error_t init_encoder();
+
 };
 
 // TODO: merge this code with SrsFragmentedMp4 in dash
@@ -342,7 +348,7 @@ class SrsHlsFmp4Muxer
     // When publish or unpublish stream.
     virtual srs_error_t on_publish(SrsRequest* req);
 
-    virtual srs_error_t write_init_mp4(SrsFormat* format);
+    virtual srs_error_t write_init_mp4(SrsFormat* format, bool has_video, bool has_audio);
     virtual srs_error_t write_audio(SrsSharedPtrMessage* shared_audio, SrsFormat* format);
     virtual srs_error_t write_video(SrsSharedPtrMessage* shared_video, SrsFormat* format);
 

trunk/src/kernel/srs_kernel_codec.hpp

@@ -1298,6 +1298,7 @@ class SrsVideoFrame : public SrsFrame
 {
 public:
     // video specified
+    // TODO: H.264 and H.265 reused AvcFrameType and AvcFrameTrait?
     SrsVideoAvcFrameType frame_type;
     SrsVideoAvcFrameTrait avc_packet_type;
     // whether sample_units contains IDR frame.

trunk/src/kernel/srs_kernel_mp4.cpp

@@ -6646,18 +6646,22 @@ srs_error_t SrsMp4M2tsInitEncoder::write(SrsFormat* format, int v_tid, int a_tid
             moov->set_mvex(mvex);
 
             // video trex
-            SrsMp4TrackExtendsBox* v_trex = new SrsMp4TrackExtendsBox();
-            mvex->add_trex(v_trex);
+            if (format->vcodec) {
+                SrsMp4TrackExtendsBox* v_trex = new SrsMp4TrackExtendsBox();
+                mvex->add_trex(v_trex);
 
-            v_trex->track_ID = v_tid;
-            v_trex->default_sample_description_index = 1;
+                v_trex->track_ID = v_tid;
+                v_trex->default_sample_description_index = 1;
+            }
 
             // audio trex
-            SrsMp4TrackExtendsBox* a_trex = new SrsMp4TrackExtendsBox();
-            mvex->add_trex(a_trex);
+            if (format->acodec) {
+                SrsMp4TrackExtendsBox* a_trex = new SrsMp4TrackExtendsBox();
+                mvex->add_trex(a_trex);
 
-            a_trex->track_ID = a_tid;
-            a_trex->default_sample_description_index = 1;
+                a_trex->track_ID = a_tid;
+                a_trex->default_sample_description_index = 1;
+            }
         }
 
         if ((err = srs_mp4_write_box(writer, moov.get())) != srs_success) {