Given that the remote host has ffmpeg installed, use the following to get remote access to the camera:

ssh user@server "ffmpeg -f v4l2 -i /dev/video0 -f matroska -codec:v libx264 -preset veryfast -tune zerolatency -" | mpv -

If you want closer to “zerolatency”, try this instead:

ssh user@server '
ffmpeg -hide_banner -loglevel error \
  -f v4l2 -framerate 30 -video_size 1280x720 -i /dev/video0 \
  -an \
  -c:v libx264 -preset ultrafast -tune zerolatency \
  -x264-params keyint=30:min-keyint=30:scenecut=0:bframes=0:rc-lookahead=0 \
  -fflags nobuffer -flags low_delay \
  -f mpegts -muxdelay 0 -muxpreload 0 -
' | mpv --profile=low-latency --untimed --cache=no --demuxer-max-bytes=1M --demuxer-readahead-secs=0 -