接触iOS硬解码是从2017上半年开始的,在接触硬解码之前都是集成各种接收数据到渲染的框架,各种坑填都填不完,就直接改用硬解码了,tcp网络码流获取方面使用的是CocoaAsyncSocket网络框架,rtsp摄像头码流获取是通过FFmpeg获取的,这里就不将代码贴出来了,需要的话评论区留言,接下来直接上摄像头h264码流图
上图:这是一段摄像头的sps ,pps,I帧粘包都一块的数据图
- // H264Decoder.m
- #import "H264Decoder.h"
- const uint8_t KStartCode[4] = {0, 0, 0, 1};
- @implementation H264Decoder
- - (instancetype)init
- {
- if (self = [super init]) {
- sps = malloc(64);
- pps = malloc(64);
- first = true;
- _decompressionSession = NULL;
- _recordNum = 1;
- }
- return self;
- }
- - (NSLock *)theLock
- {
- if (!_theLock) {
- _theLock = [[NSLock alloc] init];
- }
- return _theLock;
- }
- -(void)decodeFrame:(uint8_t *)frame withSize:(uint32_t)frameSize
- {
- //NSData *datas = [NSData dataWithBytes:frame length:frameSize];
- if(frameSize <= 0){
- return;
- }
- // FILE* fp = fopen("/Users/ff/Desktop/h264.h264","ab+");
- // if(fp != NULL)
- // {
- // NSLog(@"写数据");
- // fwrite(frame, 1, frameSize, fp);
- // fclose(fp);
- // }
- int readCount = 0;
- int count=0,len=0;
- char oldValue=0x00;
- // _cellTag = tag;
- OSStatus status = 0;
- uint8_t *data = NULL;
- int startCodeIndex = 0;
- long blockLength = 0;
- int IDRStastus = 0;
- //解码前
- CMSampleBufferRef sampleBuffer = NULL;
- //解码后
- CMBlockBufferRef blockBuffer = NULL;
- //
- int nalu_types = (frame[startCodeIndex + 4] & 0x1F);
- if(nalu_types != 1){
- do{//处理粘包数据
- char value=frame[readCount];
- //NSLog(@"value %02u",value);
- readCount++;
- //printf("read data buf value %u \n",value);
- if(value==0x00 || value==0x01||readCount==frameSize){
- if(readCount==frameSize){
- sendBuf[len++]=value;
- }
- if((count==3&&value==0x01)||readCount==frameSize){
- if(len>4||readCount==frameSize){
- {
- nalu_types = (sendBuf[4]&0x1F);
- int length=len-3;
- if(readCount==frameSize){
- length=len;
- }
- //NSLog(@"当前下标---%ld-----数据类型%d",tag,nalu_types);
- if(nalu_types==7){
- if(_recordNum == 1 && sps != NULL){
- _spsSize = length-4;
- memcpy(sps, &sendBuf[4], _spsSize);
- }
- }else if(nalu_types==8){
- if(_recordNum == 1 && pps != NULL){
- _ppsSize = length-4;
- memcpy(pps, &sendBuf[4], _ppsSize);
- //first &&
- if( _spsSize > 0 && _ppsSize > 0)
- {
- uint8_t* parameterSetPointers[2] = {sps, pps};
- size_t parameterSetSizes[2] = {_spsSize, _ppsSize};
- status = CMVideoFormatDescriptionCreateFromH264ParameterSets(kCFAllocatorDefault, 2,
- (const uint8_t *const*)
- parameterSetPointers,
- parameterSetSizes, 4,
- &_formatDesc);
- if((status == noErr) && (_decompressionSession == NULL))
- {
- [self createDecompSession];
- }
- first = false;
- }
- _recordNum = 2;
- }
- }else if(nalu_types==6){
- }else if(nalu_types==5){
- IDRStastus = 1;
- blockLength = length;
- data = malloc(blockLength);
- data = memcpy(data, &sendBuf[0], blockLength);
- uint32_t dataLength32 = htonl (blockLength-4);
- memcpy (data, &dataLength32, sizeof (uint32_t));
- // NSData *datas = [NSData dataWithBytes:data length:blockLength];
- // NSLog(@"1----%@",datas);
- //2020-02-26 10:29:45.017755+0800 FFMediaDemo[2132:1825923] {length = 1251, bytes = 0x000004df 25b82002 bff62e31 21ce2800 ... 891ecdad 935bbe9f }----
- status = CMBlockBufferCreateWithMemoryBlock(NULL, data,
- blockLength,
- kCFAllocatorNull, NULL,
- 0,
- blockLength,
- 0, &blockBuffer);
- if(status != kCMBlockBufferNoErr){
- NSLog(@"\t\t BlockBufferCreation: \t %@", (status == kCMBlockBufferNoErr) ? @"successful!" : @"failed...");
- return;
- }
- }else if(nalu_types==1){
- blockLength = length;
- data = malloc(blockLength);
- data = memcpy(data, &sendBuf[0], blockLength);
- uint32_t dataLength32 = htonl (blockLength-4);
- memcpy (data, &dataLength32, sizeof (uint32_t));
- status = CMBlockBufferCreateWithMemoryBlock(NULL, data,
- blockLength,
- kCFAllocatorNull, NULL,
- 0,
- blockLength,
- 0, &blockBuffer);
- }else if(nalu_types==9){
- }
- if(readCount!=frameSize){
- int nextNalu_types = (frame[readCount]&0x1F);
- if(nextNalu_types == 5){
- nalu_types = nextNalu_types;
- break;
- }
- if(nextNalu_types == 1){
- nalu_types = nextNalu_types;
- break;
- }
- }
- }
- len=3;
- sendBuf[0]=0x00;
- sendBuf[1]=0x00;
- sendBuf[2]=0x00;
- }
- count=0;
- }else{
- if(value==0x00 && oldValue==0x00){
- count++;
- if(count>3){
- count-=1;
- }
- }else{
- if(value==0x00){
- count=1;
- }else{
- count=0;
- }
- }
- }
- }
- sendBuf[len]=value;
- oldValue=value;
- len++;
- }while(readCount<frameSize);
- }
- if(nalu_types == 5 && IDRStastus != 1){
- //NSLog(@"当前帧类型:%d-----下一帧类型",nalu_types);
- blockLength = frameSize-(readCount-4);
- data = malloc(blockLength);
- memcpy(data, &frame[readCount-4], blockLength);
- uint32_t dataLength32 = htonl (blockLength-4);
- memcpy (data, &dataLength32, sizeof (uint32_t));
- // NSData *datas = [NSData dataWithBytes:data length:blockLength];
- // NSLog(@"2----%@",datas);
- status = CMBlockBufferCreateWithMemoryBlock(NULL, data,
- blockLength,
- kCFAllocatorNull, NULL,
- 0,
- blockLength,
- 0, &blockBuffer);
- }
- if(nalu_types == 1){
- blockLength = frameSize-(readCount-4);
- data = malloc(blockLength);
- if(data == NULL)return;
- int newRC = readCount-4;
- if(newRC <= 0){
- newRC = 0;
- }
- memcpy(data, &frame[newRC], blockLength);
- uint32_t dataLength32 = htonl (blockLength-4);
- memcpy (data, &dataLength32, sizeof (uint32_t));
- status = CMBlockBufferCreateWithMemoryBlock(NULL, data,
- blockLength,
- kCFAllocatorNull, NULL,
- 0,
- blockLength,
- 0, &blockBuffer);
- }
- if(status == kCMBlockBufferNoErr)
- {
- status = CMSampleBufferCreate(NULL,
- blockBuffer,
- TRUE, 0, 0,
- _formatDesc,
- 1, 0, NULL, 0,
- NULL, &sampleBuffer);
- }
- if(status == kCMBlockBufferNoErr)
- {
- [self.theLock lock];
- [self render:sampleBuffer];
- [self.theLock unlock];
- }
- if (NULL != blockBuffer) {
- CFRelease(blockBuffer);
- blockBuffer = NULL;
- }
- if (NULL != sampleBuffer) {
- CFRelease(sampleBuffer);
- sampleBuffer = NULL;
- }
- [self relaseData:data];
- }
- -(void)relaseData:(uint8_t*) tmpData{
- if (NULL != tmpData)
- {
- free (tmpData);
- tmpData = NULL;
- }
- }
- -(void) createDecompSession
- {
- if(_decompressionSession) {
- VTDecompressionSessionWaitForAsynchronousFrames(_decompressionSession);
- VTDecompressionSessionInvalidate(_decompressionSession);
- CFRelease(_decompressionSession);
- _decompressionSession = NULL;
- }
- VTDecompressionOutputCallbackRecord callBackRecord;
- callBackRecord.decompressionOutputCallback = decompressionSessionDecodeFrameCallback;
- callBackRecord.decompressionOutputRefCon = (__bridge void *)self;
- CMVideoDimensions dimensions = CMVideoFormatDescriptionGetDimensions(_formatDesc);
- uint32_t width = dimensions.width;
- uint32_t height = dimensions.height;
- NSDictionary* destinationPixelBufferAttributes = @{
- (id)kCVPixelBufferPixelFormatTypeKey : [NSNumber numberWithInt:kCVPixelFormatType_420YpCbCr8BiPlanarFullRange],
- //硬解必须是 kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
- //或者是kCVPixelFormatType_420YpCbCr8Planar
- //因为iOS是 nv12 其他是nv21
- (id)kCVPixelBufferWidthKey : [NSNumber numberWithInt:width],
- (id)kCVPixelBufferHeightKey : [NSNumber numberWithInt:height],
- //这里宽高和编码反的
- (id)kCVPixelBufferOpenGLCompatibilityKey : [NSNumber numberWithBool:YES]
- , (id)kCVPixelBufferOpenGLESCompatibilityKey : [NSNumber numberWithBool:YES]
- };
- OSStatus status = VTDecompressionSessionCreate(kCFAllocatorDefault,
- _formatDesc,
- NULL,
- (__bridge CFDictionaryRef _Nullable)(destinationPixelBufferAttributes),
- &callBackRecord,
- &_decompressionSession);
- // NSLog(@"SessionStatus---%d-----%ld",status,_cellTag);
- VTSessionSetProperty(_decompressionSession, kVTDecompressionPropertyKey_ThreadCount, (__bridge CFTypeRef)[NSNumber numberWithInt:1]);
- VTSessionSetProperty(_decompressionSession, kVTDecompressionPropertyKey_RealTime, kCFBooleanTrue);
- }
- void decompressionSessionDecodeFrameCallback(void *decompressionOutputRefCon,
- void *sourceFrameRefCon,
- OSStatus status,
- VTDecodeInfoFlags infoFlags,
- CVPixelBufferRef pixelBufferRef,
- CMTime presentationTimeStamp,
- CMTime presentationDuration)
- {
- // NSLog(@"当前下标---%ld",weakSelf.cellTag);
- if (status != noErr || !pixelBufferRef) {
- NSLog(@"Error decompresssing frame at time: %.3f error: %d infoFlags: %u", (float)presentationTimeStamp.value/presentationTimeStamp.timescale, (int)status, (unsigned int)infoFlags);
- return;
- }
- __weak H264Decoder *weakSelf = (__bridge H264Decoder *)decompressionOutputRefCon;
- if(weakSelf.delegate && [weakSelf.delegate respondsToSelector:@selector(displayDecodedFrame:cellTag:)]){
- [weakSelf.delegate displayDecodedFrame:pixelBufferRef];
- }
- // [weakSelf pixelBufferRefTransformationSampleBufferRef:pixelBufferRef weakSelf:weakSelf cellTag:weakSelf.cellTag];
- // [weakSelf.delegate getDecodeImageData:pixelBufferRef cellTag:weakSelf.cellTag];
- // if (NULL != pixelBufferRef) {
- // CVPixelBufferRelease(pixelBufferRef);
- // pixelBufferRef = NULL;
- // }
- }
- - (void)pixelBufferRefTransformationSampleBufferRef:(CVPixelBufferRef)pixelBufferRef weakSelf:(H264Decoder *)weakSelf cellTag:(NSInteger)tag
- {
- //不设置具体时间信息
- CMSampleTimingInfo timing = {kCMTimeInvalid, kCMTimeInvalid, kCMTimeInvalid};
- //获取视频信息
- CMVideoFormatDescriptionRef videoInfo = NULL;
- CMVideoFormatDescriptionCreateForImageBuffer(NULL, pixelBufferRef, &videoInfo);
- NSParameterAssert(videoInfo != NULL);
- CMSampleBufferRef sampleBuffer = NULL;
- CMSampleBufferCreateForImageBuffer(kCFAllocatorDefault,pixelBufferRef, true, NULL, NULL, videoInfo, &timing, &sampleBuffer);
- NSParameterAssert(sampleBuffer != NULL);
- CFRelease(videoInfo);
- CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, YES);
- CFMutableDictionaryRef dict = (CFMutableDictionaryRef)CFArrayGetValueAtIndex(attachments, 0);
- CFDictionarySetValue(dict, kCMSampleAttachmentKey_DisplayImmediately, kCFBooleanTrue);
- if(weakSelf.delegate && [weakSelf.delegate respondsToSelector:@selector(getDecodeImageData:cellTag:)]){
- [weakSelf.delegate getDecodeImageData:sampleBuffer];
- }
- if (NULL != sampleBuffer) {
- CFRelease(sampleBuffer);
- sampleBuffer = NULL;
- }
- }
- - (void) render:(CMSampleBufferRef)sampleBuffer
- {
- if(_decompressionSession){
- VTDecodeFrameFlags flags = kVTDecodeFrame_EnableAsynchronousDecompression;
- VTDecodeInfoFlags flagOut;
- OSStatus decodeStatus;
- {
- decodeStatus = VTDecompressionSessionDecodeFrame(_decompressionSession, sampleBuffer, flags,
- &sampleBuffer, &flagOut);
- }
- if(decodeStatus == noErr){
- }else if(decodeStatus == kVTInvalidSessionErr) {
- // NSLog(@"IOS8VT: Invalid session, reset decoder session");
- [self resetH264Decoder];
- } else if(decodeStatus == kVTParameterErr) {
- //[self resetH264Decoder];
- // NSLog(@"IOS8VT: decode failed status=%d(Bad data)", decodeStatus);
- } else if(decodeStatus == kVTVideoDecoderBadDataErr) {
- // NSLog(@"IOS8VT: decode failed status=%d(Bad data)", decodeStatus);
- } else if(decodeStatus != noErr) {
- // NSLog(@"IOS8VT: decode failed status=%d", decodeStatus);
- } else if (decodeStatus == kVTVideoDecoderMalfunctionErr){
- }
- }
- }
- - (void)resetH264Decoder
- {
- if(_decompressionSession) {
- // if(_decompressionSession != nil){
- // VTDecompressionSessionWaitForAsynchronousFrames(_decompressionSession);
- // VTDecompressionSessionInvalidate(_decompressionSession);
- // CFRelease(_decompressionSession);
- // }
- _decompressionSession = NULL;
- }
- NSDictionary* destinationPixelBufferAttributes = @{
- (id)kCVPixelBufferPixelFormatTypeKey : [NSNumber numberWithInt:kCVPixelFormatType_420YpCbCr8BiPlanarFullRange],
- //硬解必须是 kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
- // 或者是kCVPixelFormatType_420YpCbCr8Planar
- //因为iOS是 nv12 其他是nv21
- (id)kCVPixelBufferWidthKey : [NSNumber numberWithInt:h264outputWidth],
- (id)kCVPixelBufferHeightKey : [NSNumber numberWithInt:h264outputHeight],
- //这里款高和编码反的
- (id)kCVPixelBufferOpenGLCompatibilityKey : [NSNumber numberWithBool:YES]
- };
- VTDecompressionOutputCallbackRecord callBackRecord;
- callBackRecord.decompressionOutputCallback = decompressionSessionDecodeFrameCallback;
- callBackRecord.decompressionOutputRefCon =(__bridge void *)self;;
- VTDecompressionSessionCreate(kCFAllocatorSystemDefault,
- _formatDesc,
- NULL, (__bridge CFDictionaryRef _Nullable)(destinationPixelBufferAttributes),
- &callBackRecord,
- &_decompressionSession);
- }
- - (void)clearH264Deocder:(dispatch_queue_t)queue{
- [self.theLock lock];
- dispatch_time_t delayTime = dispatch_time(DISPATCH_TIME_NOW, (int64_t)(0.5/*延迟执行时间*/ * NSEC_PER_SEC));
- dispatch_after(delayTime, queue, ^{
- if(_decompressionSession){
- VTDecompressionSessionInvalidate(_decompressionSession);
- CFRelease(_decompressionSession);
- _decompressionSession = NULL;
- }
- if (_formatDesc) {
- CFRelease(_formatDesc);
- _formatDesc = NULL;
- }
- if (sps != NULL) {
- free(sps);
- sps = NULL;
- _spsSize = 0;
- }
- if (pps != NULL) {
- free(pps);
- pps = NULL;
- _ppsSize = 0;
- }
- });
- [self.theLock unlock];
- }
- NSString * const naluTypesStrings[] =
- {
- @"0: Unspecified (non-VCL)",
- @"1: Coded slice of a non-IDR picture (VCL)", // P frame
- @"2: Coded slice data partition A (VCL)",
- @"3: Coded slice data partition B (VCL)",
- @"4: Coded slice data partition C (VCL)",
- @"5: Coded slice of an IDR picture (VCL)", // I frame
- @"6: Supplemental enhancement information (SEI) (non-VCL)",
- @"7: Sequence parameter set (non-VCL)", // SPS parameter
- @"8: Picture parameter set (non-VCL)", // PPS parameter
- @"9: Access unit delimiter (non-VCL)",
- @"10: End of sequence (non-VCL)",
- @"11: End of stream (non-VCL)",
- @"12: Filler data (non-VCL)",
- @"13: Sequence parameter set extension (non-VCL)",
- @"14: Prefix NAL unit (non-VCL)",
- @"15: Subset sequence parameter set (non-VCL)",
- @"16: Reserved (non-VCL)",
- @"17: Reserved (non-VCL)",
- @"18: Reserved (non-VCL)",
- @"19: Coded slice of an auxiliary coded picture without partitioning (non-VCL)",
- @"20: Coded slice extension (non-VCL)",
- @"21: Coded slice extension for depth view components (non-VCL)",
- @"22: Reserved (non-VCL)",
- @"23: Reserved (non-VCL)",
- @"24: STAP-A Single-time aggregation packet (non-VCL)",
- @"25: STAP-B Single-time aggregation packet (non-VCL)",
- @"26: MTAP16 Multi-time aggregation packet (non-VCL)",
- @"27: MTAP24 Multi-time aggregation packet (non-VCL)",
- @"28: FU-A Fragmentation unit (non-VCL)",
- @"29: FU-B Fragmentation unit (non-VCL)",
- @"30: Unspecified (non-VCL)",
- @"31: Unspecified (non-VCL)",
- };
- @end
从传入数据到分包在到初始化解码,配置解码参数,解码完成回调都在上述代码中,