探索 Rootless Containers：应用于 OJ 的下一代技术

darkyzhou@dev ~> skopeo copy docker://docker.io/library/centos:latest oci:/tmp/centos:latest
darkyzhou@dev ~> umoci unpack --rootless --image /tmp/centos:latest centos
   • rootless{usr/bin/newgidmap} ignoring (usually) harmless EPERM on setxattr "security.capability"
   • rootless{usr/bin/newuidmap} ignoring (usually) harmless EPERM on setxattr "security.capability"
   • rootless{usr/bin/ping} ignoring (usually) harmless EPERM on setxattr "security.capability"
   • rootless{usr/sbin/arping} ignoring (usually) harmless EPERM on setxattr "security.capability"
   • rootless{usr/sbin/clockdiff} ignoring (usually) harmless EPERM on setxattr "security.capability"
darkyzhou@dev ~> ls centos/rootfs/
bin@  dev/  etc/  home/  lib@  lib64@  lost+found/  media/  mnt/  opt/  proc/  root/  run/  sbin@  srv/  sys/  tmp/  usr/  var/

darkyzhou@dev ~> unshare -Ur
Welcome to fish, the friendly interactive shell
Type `help` for instructions on how to use fish
root@dev ~#

root@ubuntu:~# apt update
Reading package lists... Done
...
E: Could not open lock file /var/lib/apt/lists/lock - open (13: Permission denied)
E: Unable to lock directory /var/lib/apt/lists/
W: Problem unlinking the file /var/cache/apt/pkgcache.bin - RemoveCaches (13: Permission denied)
W: Problem unlinking the file /var/cache/apt/srcpkgcache.bin - RemoveCaches (13: Permission denied)

ubuntu@ubuntu:~$ unshare -Urmipun --fork
root@ubuntu:~# df -h
Filesystem      Size  Used Avail Use% Mounted on
/dev/mmcblk0p2   29G  9.1G   19G  33% /
udev            1.9G     0  1.9G   0% /dev
tmpfs           1.9G     0  1.9G   0% /dev/shm
tmpfs           380M  4.2M  376M   2% /run
tmpfs           5.0M     0  5.0M   0% /run/lock
...

root@dev ~/centos# cd centos/rootfs/
root@dev ~/c/rootfs# mount --rbind . .

root@dev ~/c/rootfs# findmnt
TARGET                          SOURCE                      FSTYPE     OPTIONS
/                               /dev/mapper/ubuntu--vg-ubuntu--lv
│                                                           ext4       rw,relatime
├─省略中间内容
└─/home/darkyzhou/centos/rootfs /dev/mapper/ubuntu--vg-ubuntu--lv[/home/darkyzhou/centos/rootfs]
                                                            ext4       rw,relatime

root@dev ~/c/rootfs# export NEW_ROOT=/home/darkyzhou/centos/rootfs
root@dev ~/c/rootfs# mount -t proc -o noexec,nosuid,nodev proc $NEW_ROOT/proc
root@dev ~/c/rootfs# mount -t tmpfs -o noexec,strictatime,mode=0755 dev $NEW_ROOT/dev
root@dev ~/c/rootfs# mkdir $NEW_ROOT/dev/shm
root@dev ~/c/rootfs# mount -t tmpfs -o noexec,nosuid,nodev,mode=1777,size=65536k shm $NEW_ROOT/dev/shm
root@dev ~/c/rootfs# mkdir $NEW_ROOT/dev/mqueue
root@dev ~/c/rootfs# mount -t mqueue -o noexec,nosuid,nodev mqueue $NEW_ROOT/dev/mqueue
root@dev ~/c/rootfs# mount -t sysfs -o noexec,nosuid,nodev,ro sys $NEW_ROOT/sys
root@dev ~/c/rootfs# mkdir $NEW_ROOT/dev/pts
root@dev ~/c/rootfs# mount -t devpts -o noexec,nosuid,newinstance,ptmxmode=0666,mode=620 devpts $NEW_ROOT/dev/pts

mount: /home/ubuntu/new_root/dev/pts: wrong fs type, bad option, bad superblock on devpts, missing codepage or helper program, or other error.

root@dev ~/c/rootfs# touch $NEW_ROOT/dev/{null,zero,full,tty,random,urandom}
root@dev ~/c/rootfs# mount --bind /dev/null $NEW_ROOT/dev/null
root@dev ~/c/rootfs# mount --bind /dev/zero $NEW_ROOT/dev/zero
root@dev ~/c/rootfs# mount --bind /dev/full $NEW_ROOT/dev/full
root@dev ~/c/rootfs# mount --bind /dev/tty $NEW_ROOT/dev/tty
root@dev ~/c/rootfs# mount --bind /dev/random $NEW_ROOT/dev/random
root@dev ~/c/rootfs# mount --bind /dev/urandom $NEW_ROOT/dev/urandom
root@dev ~/c/rootfs# ln -s /dev/ptmx $NEW_ROOT/dev/ptmx
root@dev ~/c/rootfs# ln -s /proc/self/fd $NEW_ROOT/dev/fd
root@dev ~/c/rootfs# ln -s /proc/self/fd/0 $NEW_ROOT/dev/stdin
root@dev ~/c/rootfs# ln -s /proc/self/fd/1 $NEW_ROOT/dev/stdout
root@dev ~/c/rootfs# ln -s /proc/self/fd/2 $NEW_ROOT/dev/stderr

root@dev ~/c/rootfs# cd
root@dev ~# mkdir $NEW_ROOT/old_root
root@dev ~# pivot_root $NEW_ROOT $NEW_ROOT/old_root/
root@dev ~# cd /
root@dev /# umount -l old_root/
root@dev /# rm -r old_root/
root@dev /# exec bash
bash-4.4# PS1='my-container => '
my-container =>

my-container => ls
bin@  dev/  etc/  home/  lib@  lib64@  lost+found/  media/  mnt/  old_root/  opt/  proc/  root/  run/  sbin@  srv/  sys/  tmp/  usr/  var/
my-container => cat /etc/centos-release
CentOS Linux release 8.4.2105

my-container => findmnt
TARGET           SOURCE                                                           FSTYPE   OPTIONS
/                /dev/mapper/ubuntu--vg-ubuntu--lv[/home/darkyzhou/centos/rootfs] ext4     rw,relatime
|-/proc          proc                                                             proc     rw,nosuid,nodev,noexec,relatime
|-/dev           dev                                                              tmpfs    rw,noexec,mode=755,uid=1000,gid=1000,inode64
| |-/dev/shm     shm                                                              tmpfs    rw,nosuid,nodev,noexec,relatime,size=65536k,uid=1000,gid=1000,inode64
| |-/dev/mqueue  mqueue                                                           mqueue   rw,nosuid,nodev,noexec,relatime
| |-/dev/pts     devpts                                                           devpts   rw,nosuid,noexec,relatime,mode=620,ptmxmode=666
| |-/dev/null    udev[/null]                                                      devtmpfs rw,nosuid,relatime,size=8158036k,nr_inodes=2039509,mode=755,inode64
| |-/dev/zero    udev[/zero]                                                      devtmpfs rw,nosuid,relatime,size=8158036k,nr_inodes=2039509,mode=755,inode64
| |-/dev/full    udev[/full]                                                      devtmpfs rw,nosuid,relatime,size=8158036k,nr_inodes=2039509,mode=755,inode64
| |-/dev/tty     udev[/tty]                                                       devtmpfs rw,nosuid,relatime,size=8158036k,nr_inodes=2039509,mode=755,inode64
| |-/dev/random  udev[/random]                                                    devtmpfs rw,nosuid,relatime,size=8158036k,nr_inodes=2039509,mode=755,inode64
| `-/dev/urandom udev[/urandom]                                                   devtmpfs rw,nosuid,relatime,size=8158036k,nr_inodes=2039509,mode=755,inode64
`-/sys           sys                                                              sysfs    ro,nosuid,nodev,noexec,relatime

{
  "ociVersion": "1.0.0",
  // 此项描述的是初始进程（PID 为 1）的各种设置
  "process": {
    "terminal": true,
    "user": {
      "uid": 0,
      "gid": 0
    },
    "args": [
      "/bin/bash"
    ],
    // 一些基本的环境变量
    "env": [
      "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
      "TERM=xterm",
      "HOME=/root"
    ],
    "cwd": "/",
    // compatibility 配置
    "capabilities": {
      "bounding": [
        "CAP_AUDIT_WRITE",
        "CAP_KILL",
        "CAP_NET_BIND_SERVICE"
      ],
      "effective": [
        "CAP_AUDIT_WRITE",
        "CAP_KILL",
        "CAP_NET_BIND_SERVICE"
      ],
      "inheritable": [
        "CAP_AUDIT_WRITE",
        "CAP_KILL",
        "CAP_NET_BIND_SERVICE"
      ],
      "permitted": [
        "CAP_AUDIT_WRITE",
        "CAP_KILL",
        "CAP_NET_BIND_SERVICE"
      ],
      "ambient": [
        "CAP_AUDIT_WRITE",
        "CAP_KILL",
        "CAP_NET_BIND_SERVICE"
      ]
    },
    // rlimit 配置
    "rlimits": [
      {
        "type": "RLIMIT_NOFILE",
        "hard": 1024,
        "soft": 1024
      }
    ],
    // 这是一项很重要的设置，与安全有关，具体可见：
    // https://www.kernel.org/doc/Documentation/prctl/no_new_privs.txt
    "noNewPrivileges": true
  },
  "root": {
    "path": "rootfs"
  },
  "hostname": "umoci-default",
  // 这里的挂载配置与前文说的差不多，都是一些重要的文件系统
  "mounts": [
    {
      "destination": "/proc",
      "type": "proc",
      "source": "proc"
    },
    {
      "destination": "/dev",
      "type": "tmpfs",
      "source": "tmpfs",
      "options": [
        "nosuid",
        "strictatime",
        "mode=755",
        "size=65536k"
      ]
    },
    {
      "destination": "/dev/pts",
      "type": "devpts",
      "source": "devpts",
      "options": [
        "nosuid",
        "noexec",
        "newinstance",
        "ptmxmode=0666",
        "mode=0620"
      ]
    },
    {
      "destination": "/dev/shm",
      "type": "tmpfs",
      "source": "shm",
      "options": [
        "nosuid",
        "noexec",
        "nodev",
        "mode=1777",
        "size=65536k"
      ]
    },
    {
      "destination": "/dev/mqueue",
      "type": "mqueue",
      "source": "mqueue",
      "options": [
        "nosuid",
        "noexec",
        "nodev"
      ]
    },
    {
      "destination": "/sys",
      "type": "bind",
      "source": "/sys",
      "options": [
        "rbind",
        "nosuid",
        "noexec",
        "nodev",
        "ro"
      ]
    },
    {
      "destination": "/etc/resolv.conf",
      "type": "bind",
      "source": "/etc/resolv.conf",
      "options": [
        "nodev",
        "noexec",
        "nosuid",
        "rbind",
        "ro"
      ]
    }
  ],
  // 这些设置只是为了标识
  "annotations": {
    "org.label-schema.build-date": "20210915",
    "org.label-schema.license": "GPLv2",
    "org.label-schema.name": "CentOS Base Image",
    "org.label-schema.schema-version": "1.0",
    "org.label-schema.vendor": "CentOS",
    "org.opencontainers.image.architecture": "amd64",
    "org.opencontainers.image.author": "",
    "org.opencontainers.image.created": "2021-09-15T18:20:05.184694267Z",
    "org.opencontainers.image.exposedPorts": "",
    "org.opencontainers.image.os": "linux",
    "org.opencontainers.image.stopSignal": ""
  },
  "linux": {
    // 下面两个 mapping 是使用用户命名空间的关键
    // 它将容器内的 0 号 PID 和 GID 映射到宿主系统的普通用户和用户组
    "uidMappings": [
      {
        "containerID": 0,
        "hostID": 1000,
        "size": 1
      }
    ],
    "gidMappings": [
      {
        "containerID": 0,
        "hostID": 1000,
        "size": 1
      }
    ],
    // umoci 默认只创建了以下五个命名空间
    // 值得注意的是它并没有创建网络命名空间，因此容器创建之后可以正常上网
    // 这是因为容器和宿主系统使用的网络栈、网卡等都是一样的
    // 但需要注意，容器内部需要提供正确的 /etc/resolv.conf 文件
    "namespaces": [
      {
        "type": "pid"
      },
      {
        "type": "ipc"
      },
      {
        "type": "uts"
      },
      {
        "type": "mount"
      },
      {
        "type": "user"
      }
    ],
    // 以下文件被设置为不可读，主要是为了安全和隔离性考虑
    // 例如，/proc/kcore 文件代表了系统的物理文件，使得容器内部有可能访问到不属于容器的内存
    // 此文件默认会被上面 mount 的 proc 文件系统提供，因此这里需要屏蔽掉，故称为“masked”
    "maskedPaths": [
      "/proc/kcore",
      "/proc/latency_stats",
      "/proc/timer_list",
      "/proc/timer_stats",
      "/proc/sched_debug",
      "/sys/firmware",
      "/proc/scsi"
    ],
    // 以下文件被设置为只读（不可修改），它们主要是一些内核的信息
    // 不同容器之间共享同一个内核，因此将它们设置为不可修改是为了防止容器之间相互影响
    "readonlyPaths": [
      "/proc/asound",
      "/proc/bus",
      "/proc/fs",
      "/proc/irq",
      "/proc/sys",
      "/proc/sysrq-trigger"
    ]
  }
}

darkyzhou@dev ~> runc run -b centos/ my-container
[root@umoci-default /]#