From 015b25090544fd090130fb5ba3187ca852b44b71 Mon Sep 17 00:00:00 2001 From: Thomas Stachl Date: Thu, 15 Jul 2021 15:35:54 -0700 Subject: [PATCH] serve/docker: build docker plugin for multiple platforms #5668 Fixes #5462 Co-authored-by: Ivan Andreev --- .../build_publish_release_docker_image.yml | 38 ++++++---------- Makefile | 45 +++++++++---------- cmd/serve/docker/contrib/plugin/Dockerfile | 35 --------------- contrib/docker-plugin/managed/Dockerfile | 25 +++++++++++ .../docker-plugin/managed}/config.json | 4 +- .../systemd/docker-volume-rclone.service | 0 .../systemd/docker-volume-rclone.socket | 0 docs/content/docker.md | 41 ++++++++++++++--- 8 files changed, 95 insertions(+), 93 deletions(-) delete mode 100644 cmd/serve/docker/contrib/plugin/Dockerfile create mode 100644 contrib/docker-plugin/managed/Dockerfile rename {cmd/serve/docker/contrib/plugin => contrib/docker-plugin/managed}/config.json (94%) rename {cmd/serve/docker/contrib => contrib/docker-plugin}/systemd/docker-volume-rclone.service (100%) rename {cmd/serve/docker/contrib => contrib/docker-plugin}/systemd/docker-volume-rclone.socket (100%) diff --git a/.github/workflows/build_publish_release_docker_image.yml b/.github/workflows/build_publish_release_docker_image.yml index 936cdc44e..286a47ad2 100644 --- a/.github/workflows/build_publish_release_docker_image.yml +++ b/.github/workflows/build_publish_release_docker_image.yml @@ -37,35 +37,23 @@ jobs: if: github.repository == 'rclone/rclone' needs: build runs-on: ubuntu-latest - name: Build and publish docker volume plugin + name: Build docker plugin job steps: - name: Checkout master uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Set plugin parameters + - name: Build and publish docker plugin shell: bash run: | - GITHUB_REF=${{ github.ref }} - - PLUGIN_IMAGE_USER=rclone - PLUGIN_IMAGE_NAME=docker-volume-rclone - PLUGIN_IMAGE_TAG=${GITHUB_REF#refs/tags/} - PLUGIN_IMAGE=${PLUGIN_IMAGE_USER}/${PLUGIN_IMAGE_NAME}:${PLUGIN_IMAGE_TAG} - PLUGIN_IMAGE_LATEST=${PLUGIN_IMAGE_USER}/${PLUGIN_IMAGE_NAME}:latest - - echo "PLUGIN_IMAGE_USER=${PLUGIN_IMAGE_USER}" >> $GITHUB_ENV - echo "PLUGIN_IMAGE_NAME=${PLUGIN_IMAGE_NAME}" >> $GITHUB_ENV - echo "PLUGIN_IMAGE_TAG=${PLUGIN_IMAGE_TAG}" >> $GITHUB_ENV - echo "PLUGIN_IMAGE=${PLUGIN_IMAGE}" >> $GITHUB_ENV - echo "PLUGIN_IMAGE_LATEST=${PLUGIN_IMAGE_LATEST}" >> $GITHUB_ENV - - name: Build image - shell: bash - run: | - make docker-plugin - - name: Push image - shell: bash - run: | - docker login -u ${{ secrets.DOCKER_HUB_USER }} -p ${{ secrets.DOCKER_HUB_PASSWORD }} - make docker-plugin-push PLUGIN_IMAGE=${PLUGIN_IMAGE} - make docker-plugin-push PLUGIN_IMAGE=${PLUGIN_IMAGE_LATEST} + VER=${GITHUB_REF#refs/tags/} + PLUGIN_USER=rclone + docker login --username ${{ secrets.DOCKER_HUB_USER }} \ + --password-stdin <<< "${{ secrets.DOCKER_HUB_PASSWORD }}" + for PLUGIN_ARCH in amd64 arm64 arm/v7 ;do + export PLUGIN_USER PLUGIN_ARCH + make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-} + make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}-${VER#v} + done + make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=latest + make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=${VER#v} diff --git a/Makefile b/Makefile index 3541592fa..7d4907b93 100644 --- a/Makefile +++ b/Makefile @@ -258,34 +258,31 @@ winzip: zip -9 rclone-$(TAG).zip rclone.exe # docker volume plugin -PLUGIN_IMAGE_USER ?= rclone -PLUGIN_IMAGE_TAG ?= latest -PLUGIN_IMAGE_NAME ?= docker-volume-rclone -PLUGIN_IMAGE ?= $(PLUGIN_IMAGE_USER)/$(PLUGIN_IMAGE_NAME):$(PLUGIN_IMAGE_TAG) - -PLUGIN_BASE_IMAGE := rclone/rclone:latest +PLUGIN_USER ?= rclone +PLUGIN_TAG ?= latest +PLUGIN_BASE_TAG ?= latest +PLUGIN_ARCH ?= amd64 +PLUGIN_IMAGE := $(PLUGIN_USER)/docker-volume-rclone:$(PLUGIN_TAG) +PLUGIN_BASE := $(PLUGIN_USER)/rclone:$(PLUGIN_BASE_TAG) PLUGIN_BUILD_DIR := ./build/docker-plugin -PLUGIN_CONTRIB_DIR := ./cmd/serve/docker/contrib/plugin -PLUGIN_CONFIG := $(PLUGIN_CONTRIB_DIR)/config.json -PLUGIN_DOCKERFILE := $(PLUGIN_CONTRIB_DIR)/Dockerfile -PLUGIN_CONTAINER := docker-volume-rclone-dev-$(shell date +'%Y%m%d-%H%M%S') - -docker-plugin: docker-plugin-rootfs docker-plugin-create - -docker-plugin-image: rclone - docker build --no-cache --pull --build-arg BASE_IMAGE=${PLUGIN_BASE_IMAGE} -t ${PLUGIN_IMAGE} -f ${PLUGIN_DOCKERFILE} . - -docker-plugin-rootfs: docker-plugin-image - mkdir -p ${PLUGIN_BUILD_DIR}/rootfs - docker create --name ${PLUGIN_CONTAINER} ${PLUGIN_IMAGE} - docker export ${PLUGIN_CONTAINER} | tar -x -C ${PLUGIN_BUILD_DIR}/rootfs - docker rm -vf ${PLUGIN_CONTAINER} - cp ${PLUGIN_CONFIG} ${PLUGIN_BUILD_DIR}/config.json +PLUGIN_CONTRIB_DIR := ./contrib/docker-plugin/managed docker-plugin-create: - docker plugin rm -f ${PLUGIN_IMAGE} 2>/dev/null || true + docker buildx inspect |grep -q /${PLUGIN_ARCH} || \ + docker run --rm --privileged tonistiigi/binfmt --install all + rm -rf ${PLUGIN_BUILD_DIR} + docker buildx build \ + --no-cache --pull \ + --build-arg BASE_IMAGE=${PLUGIN_BASE} \ + --platform linux/${PLUGIN_ARCH} \ + --output ${PLUGIN_BUILD_DIR}/rootfs \ + ${PLUGIN_CONTRIB_DIR} + cp ${PLUGIN_CONTRIB_DIR}/config.json ${PLUGIN_BUILD_DIR} + docker plugin rm --force ${PLUGIN_IMAGE} 2>/dev/null || true docker plugin create ${PLUGIN_IMAGE} ${PLUGIN_BUILD_DIR} -docker-plugin-push: docker-plugin-create +docker-plugin-push: docker plugin push ${PLUGIN_IMAGE} docker plugin rm ${PLUGIN_IMAGE} + +docker-plugin: docker-plugin-create docker-plugin-push diff --git a/cmd/serve/docker/contrib/plugin/Dockerfile b/cmd/serve/docker/contrib/plugin/Dockerfile deleted file mode 100644 index eec817ceb..000000000 --- a/cmd/serve/docker/contrib/plugin/Dockerfile +++ /dev/null @@ -1,35 +0,0 @@ -ARG BASE_IMAGE=rclone/rclone:latest -ARG BUILD_PLATFORM=linux/amd64 -ARG TARGET_PLATFORM=linux/amd64 - -# temporary build image -FROM --platform=${BUILD_PLATFORM} golang:alpine AS BUILD_ENV - -COPY . /src -WORKDIR /src - -RUN apk add --no-cache make git bash && \ - CGO_ENABLED=0 \ - GOARCH=$(echo ${TARGET_PLATFORM} | cut -d '/' -f2) \ - make rclone - -# plugin image -FROM ${BASE_IMAGE} - -COPY --from=BUILD_ENV /src/rclone /usr/local/bin/rclone - -RUN mkdir -p /data/config /data/cache /mnt \ - && /usr/local/bin/rclone version - -ENV RCLONE_CONFIG=/data/config/rclone.conf -ENV RCLONE_CACHE_DIR=/data/cache -ENV RCLONE_BASE_DIR=/mnt -ENV RCLONE_VERBOSE=0 - -ENV HTTP_PROXY= -ENV HTTPS_PROXY= -ENV NO_PROXY= - -WORKDIR /data -ENTRYPOINT ["/usr/local/bin/rclone"] -CMD ["serve", "docker"] diff --git a/contrib/docker-plugin/managed/Dockerfile b/contrib/docker-plugin/managed/Dockerfile new file mode 100644 index 000000000..0f82b36aa --- /dev/null +++ b/contrib/docker-plugin/managed/Dockerfile @@ -0,0 +1,25 @@ +ARG BASE_IMAGE=rclone/rclone:latest +FROM ${BASE_IMAGE} as binaries + +# build plugin image +FROM alpine:latest + +# put rclone in /usr/bin, reserve /usr/local/bin for plugin wrappers +COPY --from=binaries /usr/local/bin/rclone /usr/bin/rclone + +RUN mkdir -p /data/config /data/cache /mnt \ + && apk --no-cache add ca-certificates fuse tzdata \ + && echo "user_allow_other" >> /etc/fuse.conf \ + && rclone version + +ENV RCLONE_CONFIG=/data/config/rclone.conf +ENV RCLONE_CACHE_DIR=/data/cache +ENV RCLONE_BASE_DIR=/mnt +ENV RCLONE_VERBOSE=0 + +ENV HTTP_PROXY= +ENV HTTPS_PROXY= +ENV NO_PROXY= + +WORKDIR /data +ENTRYPOINT ["rclone", "serve", "docker"] diff --git a/cmd/serve/docker/contrib/plugin/config.json b/contrib/docker-plugin/managed/config.json similarity index 94% rename from cmd/serve/docker/contrib/plugin/config.json rename to contrib/docker-plugin/managed/config.json index 22d04aad2..d00fb3e2d 100644 --- a/cmd/serve/docker/contrib/plugin/config.json +++ b/contrib/docker-plugin/managed/config.json @@ -1,6 +1,6 @@ { "description": "Rclone volume plugin for Docker", - "documentation": "https://rclone.org/", + "documentation": "https://rclone.org/docker", "interface": { "socket": "rclone.sock", "types": ["docker.volumedriver/1.0"] @@ -18,7 +18,7 @@ "network": { "type": "host" }, - "entrypoint": ["/usr/local/bin/rclone", "serve", "docker"], + "entrypoint": ["rclone", "serve", "docker"], "workdir": "/data", "args": { "name": "args", diff --git a/cmd/serve/docker/contrib/systemd/docker-volume-rclone.service b/contrib/docker-plugin/systemd/docker-volume-rclone.service similarity index 100% rename from cmd/serve/docker/contrib/systemd/docker-volume-rclone.service rename to contrib/docker-plugin/systemd/docker-volume-rclone.service diff --git a/cmd/serve/docker/contrib/systemd/docker-volume-rclone.socket b/contrib/docker-plugin/systemd/docker-volume-rclone.socket similarity index 100% rename from cmd/serve/docker/contrib/systemd/docker-volume-rclone.socket rename to contrib/docker-plugin/systemd/docker-volume-rclone.socket diff --git a/docs/content/docker.md b/docs/content/docker.md index 0b015e2fc..87c5d501d 100644 --- a/docs/content/docker.md +++ b/docs/content/docker.md @@ -53,9 +53,9 @@ sudo mkdir -p /var/lib/docker-plugins/rclone/config sudo mkdir -p /var/lib/docker-plugins/rclone/cache ``` -Install the managed rclone docker plugin: +Install the managed rclone docker plugin for your architecture (here `amd64`): ``` -docker plugin install rclone/docker-volume-rclone args="-v" --alias rclone --grant-all-permissions +docker plugin install rclone/docker-volume-rclone:amd64 args="-v" --alias rclone --grant-all-permissions docker plugin list ``` @@ -321,9 +321,36 @@ By default they must exist on host at the following locations You can [install managed plugin](https://docs.docker.com/engine/reference/commandline/plugin_install/) with default settings as follows: ``` -docker plugin install rclone/docker-volume-rclone:latest --grant-all-permissions --alias rclone +docker plugin install rclone/docker-volume-rclone:amd64 --grant-all-permissions --alias rclone ``` +The `:amd64` part of the image specification after colon is called a _tag_. +Usually you will want to install the latest plugin for your architecture. In +this case the tag will just name it, like `amd64` above. The following plugin +architectures are currently available: +- `amd64` +- `arm64` +- `arm-v7` + +Sometimes you might want a concrete plugin version, not the latest one. +Then you should use image tag in the form `:ARCHITECTURE-VERSION`. +For example, to install plugin version `v1.56.2` on architecture `arm64` +you will use tag `arm64-1.56.2` (note the removed `v`) so the full image +specification becomes `rclone/docker-volume-rclone:arm64-1.56.2`. + +We also provide the `latest` plugin tag, but since docker does not support +multi-architecture plugins as of the time of this writing, this tag is +currently an **alias for `amd64`**. +By convention the `latest` tag is the default one and can be omitted, thus +both `rclone/docker-volume-rclone:latest` and just `rclone/docker-volume-rclone` +will refer to the latest plugin release for the `amd64` platform. + +Also the `amd64` part can be omitted from the versioned rclone plugin tags. +For example, rclone image reference `rclone/docker-volume-rclone:amd64-1.56.2` +can be abbreviated as `rclone/docker-volume-rclone:1.56.2` for convenience. +However, for non-intel architectures you still have to use the full tag as +`amd64` or `latest` will fail to start. + Managed plugin is in fact a special container running in a namespace separate from normal docker containers. Inside it runs the `rclone serve docker` command. The config and cache directories are bind-mounted into the @@ -395,7 +422,7 @@ actual level assigned by rclone in the encapsulated message string. You can set custom plugin options right when you install it, _in one go_: ``` docker plugin remove rclone -docker plugin install rclone/docker-volume-rclone:latest \ +docker plugin install rclone/docker-volume-rclone:amd64 \ --alias rclone --grant-all-permissions \ args="-v --allow-other" config=/etc/rclone docker plugin inspect rclone @@ -434,8 +461,8 @@ sudo apt-get -y install fuse ``` Download two systemd configuration files: -[docker-volume-rclone.service](https://raw.githubusercontent.com/rclone/rclone/master/cmd/serve/docker/contrib/systemd/docker-volume-rclone.service) -and [docker-volume-rclone.socket](https://raw.githubusercontent.com/rclone/rclone/master/cmd/serve/docker/contrib/systemd/docker-volume-rclone.socket). +[docker-volume-rclone.service](https://raw.githubusercontent.com/rclone/rclone/master/contrib/docker-plugin/systemd/docker-volume-rclone.service) +and [docker-volume-rclone.socket](https://raw.githubusercontent.com/rclone/rclone/master/contrib/docker-plugin/systemd/docker-volume-rclone.socket). Put them to the `/etc/systemd/system/` directory: ``` @@ -489,7 +516,7 @@ Use `journalctl --unit docker` to see managed plugin output as part of the docker daemon log. Note that docker reflects plugin lines as _errors_ but their actual level can be seen from encapsulated message string. -You will usually install the latest version of managed plugin. +You will usually install the latest version of managed plugin for your platform. Use the following commands to print the actual installed version: ``` PLUGID=$(docker plugin list --no-trunc | awk '/rclone/{print$1}')