Proxmox 8 personal setup

apt update 
apt upgrade

echo 'deb http://download.proxmox.com/debian/pve bookworm pve-no-subscription' | tee -a /etc/apt/sources.list

apt install -y git build-essential pve-headers dkms jq unzip python3 python3-pip curl wget vim

wget -c https://us.download.nvidia.com/XFree86/Linux-x86_64/535.54.03/NVIDIA-Linux-x86_64-535.54.03.run


chmod +x NVIDIA-Linux-x86_64-535.54.03.run

./NVIDIA-Linux-x86_64-535.54.03.run

nvidia-smi

modprobe nvidia nvidia_uvm

update-initramfs -u -k all

echo "KERNEL==\"nvidia\", RUN+=\"/bin/bash -c '/usr/bin/nvidia-smi -L && /bin/chmod 666 /dev/nvidia*'\"" "\nKERNEL==\"nvidia_uvm\", RUN+=\"/bin/bash -c '/usr/bin/nvidia-modprobe -c0 -u && /bin/chmod 0666 /dev/nvidia-uvm*'\"" > /etc/udev/rules.d/70-nvidia.rules


now create a container 

for example edit /etc/pve/lxc/100.conf
and add
lxc.cgroup2.devices.allow: c 195:* rwm
lxc.cgroup2.devices.allow: c 243:* rwm

# Pass through device files
lxc.mount.entry: /dev/nvidia0 dev/nvidia0 none bind,optional,create=file
lxc.mount.entry: /dev/nvidiactl dev/nvidiactl none bind,optional,create=file
lxc.mount.entry: /dev/nvidia-uvm dev/nvidia-uvm none bind,optional,create=file
lxc.mount.entry: /dev/nvidia-modeset dev/nvidia-modeset none bind,optional,create=file
lxc.mount.entry: /dev/nvidia-uvm-tools dev/nvidia-uvm-tools none bind,optional,create=file


EXAMPLE CONFIG FILE
root@wnvm3:~# cat /etc/pve/lxc/100.conf 
# Allow cgroup access
# Pass through device files
arch: amd64
cores: 20
hostname: pooh-base
memory: 131072
net0: name=eth0,bridge=vmbr0,firewall=1,hwaddr=1A:BD:4D:8F:76:74,ip=dhcp,type=veth
ostype: ubuntu
rootfs: local-lvm:vm-100-disk-1,size=256G
swap: 1512
lxc.cgroup2.devices.allow: c 195:* rwm
lxc.cgroup2.devices.allow: c 234:* rwm
lxc.cgroup2.devices.allow: c 509:* rwm
lxc.mount.entry: /dev/nvidia0 dev/nvidia0 none bind,optional,create=file
lxc.mount.entry: /dev/nvidiactl dev/nvidiactl none bind,optional,create=file
lxc.mount.entry: /dev/nvidia-uvm dev/nvidia-uvm none bind,optional,create=file
lxc.mount.entry: /dev/nvidia-modeset dev/nvidia-modeset none bind,optional,create=file
lxc.mount.entry: /dev/nvidia-uvm-tools dev/nvidia-uvm-tools none bind,optional,create=file



NOTE; MODIFY THE lxc.cgroup2 line with the correct id from 

ls -l /dev/nvidia*

then install the nvidia driver on the guest the same driver version in this case at the time of writing when using ubuntu 22.04 just do

wget -c https://us.download.nvidia.com/XFree86/Linux-x86_64/535.54.03/NVIDIA-Linux-x86_64-535.54.03.run

chmod +x NVIDIA-Linux-x86_64-535.54.03.run
./NVIDIA-Linux-x86_64-535.54.03.run --no-kernel-module

for ML and cuda install
apt install nvidia-cuda-toolkit