Capturing H264 Stream with Opencv

Capture first image from h.264 video streaming using websocket - Python

One question, how is the frame/stream transmitted trough websocket? The Byte sequence looks like a nal unit, it can be PPS or SPS etc. how do you know its an IFrame for example, i dont know If cv2.imshow Support RAW H264. Look into pyav there u can open h264 raw bytes then you can try to exract one frame out of it :) let me know if you need help on pyav, Look at this post
there is an example how you can doit.

Update

Based on your comment, you need a way to parse and decode a raw h264 stream,
below is a function that give u and idea about that, you need to pass your recived bytes from websocket to this function, be aware that needs to be enough data to extract one frame.

pip install av

PyAV docs

import av

# Feed in your raw bytes from socket
def decode(raw_bytes: bytes):
    code_ctx = av.CodecContext.create("h264", "r")
    packets = code_ctx.parse(raw_bytes)
    for i, packet in enumerate(packets):
        frames = code_ctx.decode(packet)
        if frames:
            return frame[0].to_ndarray() 

You could also try to read directly with pyav the Stream with av.open("tcp://127.0.0.1:")

Update 2
Could u please test this, the issues that you have on your edit are weird, you dont need a websocket layer I thing you can read directly from raspivid

raspivid -a 12 -t 0 -w 1280 -h 720 -vf -ih -fps 30 -l -o tcp://0.0.0.0:5000

def get_first_frame(path):
    stream = av.open(path, 'r')
    for packet in stream.demux():
        frames = packet.decode()
        if frames:
            return frames[0].to_ndarray(format='bgr24')

ff = get_first_frame("tcp://0.0.0.0:5000")
cv2.imshow("Video", ff)
cv2.waitKey(0)

Capturing a SINGLE image from an rtsp H.264 video stream

To answer my own question. Instead of using read():

cap = cv2.VideoCapture('rtsp_url')

def captureimages():
    while True:
        image = cap.read()

s = threading.Thread(target=captureimages)
s.start()

if takepic == True:
    picture = image.copy()

It is more efficient to break it up in to grab() and retrieve(). Not a perfect solution, but better:

cap = cv2.VideoCapture('rtsp_url')

def captureimages():
    while True:
        cap.grab()

s = threading.Thread(target=captureimages)
s.start()

if takepic == True:
    picture = cap.retrieve()

How to get http stream from wifi camera compressed format: h.264 on python?

With a little help from comments, I could solve my problem, and works, with some initial frame loss.

import cv2
from threading import Thread
import time

url = ('http://admin:@192.168.10.1/videostream.asf?user=admin&pwd=')

class VideoStream(object):
    def __init__(self,url = ('http://admin:@192.168.10.1/videostream.asf?user=admin&pwd=')):
            self.capture = cv2.VideoCapture(url)
            self.thread = Thread(target=self.update, args=())
            self.thread.daemon = True
            self.thread.start()

    def update(self):
            while True:
                    if self.capture.isOpened():
                            (self.status, self.frame) = self.capture.read()
                    time.sleep(.01)

    def show_frame(self):
            cv2.imshow('frame', self.frame)
            key = cv2.waitKey(1)
            if key == ord('q'):
                    self.capture.release()
                    cv2.destroyAllWindows()
                    exit(1)

if __name__ == '__main__':
    video_stream = VideoStream()
    while True:
            try:
                    video_stream.show_frame()
            except AttributeError:
                    pass

Copied from this link: Video Streaming from IP Camera in Python Using OpenCV cv2.VideoCapture

How to use pyav or opencv to decode a live stream of raw H.264 data?

After hours of finding an answer for this as well. I figure this out myself.

For single thread, you can do the following:

rawData = io.BytesIO()
container = av.open(rawData, format="h264", mode='r')
cur_pos = 0
while True:
    data = await websocket.recv()
    rawData.write(data)
    rawData.seek(cur_pos)
    for packet in container.demux():
        if packet.size == 0:
            continue
        cur_pos += packet.size
        for frame in packet.decode():
            self.frames.append(frame)

That is the basic idea. I have worked out a generic version that has receiving thread and decoding thread separated. The code will also skip frames if the CPU does not keep up with the decoding speed and will start decoding from the next key frame (so you will not have the teared green screen effect). Here is the full version of the code:

import asyncio
import av
import cv2
import io
from multiprocessing import Process, Queue, Event
import time
import websockets

def display_frame(frame, start_time, pts_offset, frame_rate):
    if frame.pts is not None:
        play_time = (frame.pts - pts_offset) * frame.time_base.numerator / frame.time_base.denominator
        if start_time is not None:
            current_time = time.time() - start_time
            time_diff = play_time - current_time
            if time_diff > 1 / frame_rate:
                return False
            if time_diff > 0:
                time.sleep(time_diff)
    img = frame.to_ndarray(format='bgr24')
    cv2.imshow('Video', img)
    return True

def get_pts(frame):
    return frame.pts

def render(terminated, data_queue):
    rawData = io.BytesIO()
    cur_pos = 0
    frames_buffer = []
    start_time = None
    pts_offset = None
    got_key_frame = False
    while not terminated.is_set():
        try:
            data = data_queue.get_nowait()
        except:
            time.sleep(0.01)
            continue
        rawData.write(data)
        rawData.seek(cur_pos)
        if cur_pos == 0:
            container = av.open(rawData, mode='r')
            original_codec_ctx = container.streams.video[0].codec_context
            codec = av.codec.CodecContext.create(original_codec_ctx.name, 'r')
        cur_pos += len(data)
        dts = None
        for packet in container.demux():
            if packet.size == 0:
                continue
            dts = packet.dts
            if pts_offset is None:
                pts_offset = packet.pts
            if not got_key_frame and packet.is_keyframe:
                got_key_frame = True
            if data_queue.qsize() > 8 and not packet.is_keyframe:
                got_key_frame = False
                continue
            if not got_key_frame:
                continue
            frames = codec.decode(packet)
            if start_time is None:
                start_time = time.time()
            frames_buffer += frames
            frames_buffer.sort(key=get_pts)
            for frame in frames_buffer:
                if display_frame(frame, start_time, pts_offset, codec.framerate):
                    frames_buffer.remove(frame)
                if cv2.waitKey(1) & 0xFF == ord('q'):
                    break
        if dts is not None:
            container.seek(25000)
        rawData.seek(cur_pos)
        if cv2.waitKey(1) & 0xFF == ord('q'):
            break
    terminated.set()
    cv2.destroyAllWindows()

async def receive_encoded_video(websocket, path):
    data_queue = Queue()
    terminated = Event()
    p = Process(
        target=render,
        args=(terminated, data_queue)
    )
    p.start()
    while not terminated.is_set():
        try:
            data = await websocket.recv()
        except:
            break
        data_queue.put(data)
    terminated.set()

---

Related Topics