From 8a05c8f8be517a7b2bab98120e9cb16b8a3074cf Mon Sep 17 00:00:00 2001 From: Enzo Nocera Date: Fri, 24 Nov 2023 18:52:19 +0100 Subject: [PATCH] fix: Cross-compilation to ARM64 (#4142) * feat(docker/docs): explain how building lemmy works Signed-off-by: Enzo NOCERA * feat: add arm build * review: rename script & fix typo * feat(ci): allow cross platform compilation * feat(ci): prettier * fix(docker): fix base image name * fix: add dockerfile in CI path Signed-off-by: Enzo Nocera * fix(docker): fix runner name * fix(docker): fix builder base image * fix(docker): fix builder base image platform * fix(docker): avoid using the wrapper adduser/addgroup * feat: avoid adding the whole docker directory in the build context --------- Signed-off-by: Enzo NOCERA Signed-off-by: Enzo Nocera Co-authored-by: Dessalines Co-authored-by: Nutomic --- .dockerignore | 1 + .woodpecker.yml | 18 ++- docker/Dockerfile | 108 +++++++----------- docker/README.md | 21 ++++ .../builders/lemmy-builder-arm64/Dockerfile | 23 ++++ .../lemmy-builder-arm64/docker-build.sh | 20 ++++ 6 files changed, 119 insertions(+), 72 deletions(-) create mode 100644 docker/README.md create mode 100644 docker/builders/lemmy-builder-arm64/Dockerfile create mode 100644 docker/builders/lemmy-builder-arm64/docker-build.sh diff --git a/.dockerignore b/.dockerignore index 5982307c0..b999debba 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,7 @@ # build folders and similar which are not needed for the docker build target docker +!docker/builders/lemmy-builder-arm64/docker-build.sh api_tests ansible tests diff --git a/.woodpecker.yml b/.woodpecker.yml index 86f27b213..667e49582 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -227,17 +227,29 @@ steps: - event: push branch: main + publish_builder_arm64: + image: woodpeckerci/plugin-docker-buildx + secrets: [docker_username, docker_password] + settings: + repo: dessalines/lemmy-builder-arm64 + dockerfile: docker/builders/lemmy-builder-arm64/Dockerfile + platforms: linux/amd64 + build_args: + - RUST_RELEASE_MODE=release + tag: ${CI_COMMIT_TAG} + when: + event: tag + publish_release_docker: image: woodpeckerci/plugin-docker-buildx secrets: [docker_username, docker_password] settings: repo: dessalines/lemmy dockerfile: docker/Dockerfile - # TODO fix arm build: see: https://woodpecker.join-lemmy.org/repos/129/pipeline/2888/20 - # platforms: linux/amd64,linux/arm64 - platforms: linux/amd64 + platforms: linux/amd64, linux/arm64 build_args: - RUST_RELEASE_MODE=release + - LEMMY_VERSION=${CI_COMMIT_TAG} tag: ${CI_COMMIT_TAG} when: event: tag diff --git a/docker/Dockerfile b/docker/Dockerfile index eab612598..359c030ad 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,10 +1,18 @@ +# syntax=docker/dockerfile:1.6 ARG RUST_VERSION=1.72.1 ARG CARGO_BUILD_FEATURES=default ARG RUST_RELEASE_MODE=debug +ARG LEMMY_VERSION="dev" + ARG AMD_BUILDER_IMAGE=rust:${RUST_VERSION} -ARG ARM_BUILDER_IMAGE=blackdex/rust-musl:aarch64-musl-stable-${RUST_VERSION}-openssl3 +ARG ARM_BUILDER_IMAGE="dessalines/lemmy-builder-arm64:${LEMMY_VERSION}" + ARG AMD_RUNNER_IMAGE=debian:bookworm-slim -ARG ARM_RUNNER_IMAGE=alpine:3.18 +ARG ARM_RUNNER_IMAGE=debian:bookworm-slim + +ARG UNAME=lemmy +ARG UID=1000 +ARG GID=1000 # AMD64 builder FROM --platform=${BUILDPLATFORM} ${AMD_BUILDER_IMAGE} AS build-amd64 @@ -33,79 +41,41 @@ RUN set -ex; \ fi # ARM64 builder -# TODO currently broken -# FROM --platform=${BUILDPLATFORM} ${ARM_BUILDER_IMAGE} as build-arm64 +# NB(raskyld): this is a hack to be able to COPY --from= this image, because the variable doesn't +# seem to be expended in --form arg of COPY :( +FROM --platform=linux/amd64 ${ARM_BUILDER_IMAGE} AS build-arm64 -# ENV DEBIAN_FRONTEND=noninteractive -# ENV CARGO_HOME=/root/.cargo -# ENV PQ_LIB_DIR=/usr/local/musl/pq15/lib - -# RUN apt update && apt install -y \ -# --no-install-recommends \ -# git - -# RUN mkdir -pv "${CARGO_HOME}" && \ -# rustup set profile minimal && \ -# rustup target add aarch64-unknown-linux-musl - -# ARG CARGO_BUILD_FEATURES -# ARG RUST_RELEASE_MODE - -# WORKDIR /lemmy - -# COPY . ./ - -# # Debug build -# RUN --mount=type=cache,target=/lemmy/target set -ex; \ -# if [ "${RUST_RELEASE_MODE}" = "debug" ]; then \ -# echo "pub const VERSION: &str = \"$(git describe --tag)\";" > crates/utils/src/version.rs; \ -# cargo build --target=aarch64-unknown-linux-musl --features "${CARGO_BUILD_FEATURES}"; \ -# mv target/aarch64-unknown-linux-musl/debug/lemmy_server ./lemmy; \ -# fi - -# # Release build -# RUN set -ex; \ -# if [ "${RUST_RELEASE_MODE}" = "release" ]; then \ -# echo "pub const VERSION: &str = \"$(git describe --tag)\";" > crates/utils/src/version.rs; \ -# cargo build --target=aarch64-unknown-linux-musl --features "${CARGO_BUILD_FEATURES}" --release; \ -# mv target/aarch64-unknown-linux-musl/release/lemmy_server ./lemmy; \ -# fi - -## Final image -FROM ${AMD_RUNNER_IMAGE} +# amd64 base runner +FROM ${AMD_RUNNER_IMAGE} AS runner-linux-amd64 # Federation needs CA certificates RUN apt update && apt install -y libssl-dev libpq-dev ca-certificates -# Debian / Ubuntu non-root user creds -ARG UNAME=lemmy -ARG UID=1000 -ARG GID=1000 -RUN groupadd -g $GID -o $UNAME -RUN useradd -m -u $UID -g $GID -o -s /bin/bash $UNAME +COPY --from=build-amd64 --chmod=0755 /lemmy/lemmy /usr/local/bin + +# arm base runner +FROM ${ARM_RUNNER_IMAGE} AS runner-linux-arm64 + +RUN apt update && apt install -y ca-certificates libssl-dev libpq-dev + +COPY --from=build-arm64 --chmod=0755 /home/lemmy/lemmy_server /usr/local/bin + +# Final image that use a base runner based on the target OS and ARCH +FROM runner-${TARGETOS}-${TARGETARCH} + +LABEL org.opencontainers.image.authors="The Lemmy Authors" +LABEL org.opencontainers.image.source="https://github.com/LemmyNet/lemmy" +LABEL org.opencontainers.image.licenses="AGPL-3.0-or-later" +LABEL org.opencontainers.image.description="A link aggregator and forum for the fediverse" + +ARG UNAME +ARG GID +ARG UID + +RUN groupadd -g ${GID} -o ${UNAME} && \ + useradd -m -u ${UID} -g ${GID} -o -s /bin/bash ${UNAME} USER $UNAME -COPY --from=build-amd64 /lemmy/lemmy ./ -CMD ["./lemmy"] +ENTRYPOINT ["lemmy_server"] EXPOSE 8536 STOPSIGNAL SIGTERM - -## Arm Runner -# FROM --platform=${BUILDPLATFORM} ${ARM_RUNNER_IMAGE} - -# ARG UNAME=lemmy -# ARG UID=1000 -# ARG GID=1000 - -# RUN apk add --no-cache ca-certificates - -# COPY --from=build-arm64 --chmod=0755 /lemmy/lemmy /usr/local/bin - -# RUN addgroup -S -g ${GID} ${UNAME} && \ -# adduser -S -H -D -G ${UNAME} -u ${UID} -g "" -s /sbin/nologin ${UNAME} -# USER $UNAME - -# CMD ["lemmy"] -# EXPOSE 8536 -# STOPSIGNAL SIGTERM - diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 000000000..8972055a0 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,21 @@ +# Building Lemmy Images + +Lemmy's images are meant to be **built** on `linux/amd64`, +but they can be **executed** on both `linux/amd64` and `linux/arm64`. + +To do so we need to use a _cross toolchain_ whose goal is to build +**from** amd64 **to** arm64. + +Namely, we need to link the _lemmy_server_ with `pq` and `openssl` +shared libraries and a few others, and they need to be in `arm64`, +indeed. + +The toolchain we use to cross-compile is specifically tailored for +Lemmy's needs, see [the image repository][image-repo]. + +#### References + +- [The Linux Documentation Project on Shared Libraries][tldp-lib] + +[tldp-lib]: https://tldp.org/HOWTO/Program-Library-HOWTO/shared-libraries.html +[image-repo]: https://github.com/raskyld/lemmy-cross-toolchains diff --git a/docker/builders/lemmy-builder-arm64/Dockerfile b/docker/builders/lemmy-builder-arm64/Dockerfile new file mode 100644 index 000000000..8863b2f99 --- /dev/null +++ b/docker/builders/lemmy-builder-arm64/Dockerfile @@ -0,0 +1,23 @@ +ARG ARM_CROSS_TOOLCHAIN="ghcr.io/raskyld/aarch64-lemmy-linux-gnu:v0.1.0" + +FROM ${ARM_CROSS_TOOLCHAIN} + +# NB(raskyld): Please, do not hesitate to contact me through @raskyld@social.vivaldi.net (mastodon) +# If you have any question about the cross-toolchain +LABEL org.opencontainers.image.authors="Enzo Nocera " +LABEL org.opencontainers.image.source="https://github.com/LemmyNet/lemmy" +LABEL org.opencontainers.image.licenses="AGPL-3.0-or-later" +LABEL org.opencontainers.image.description="A prebuilt lemmy server using a cross toolchain from amd64 to aarch64/arm64" + +ARG CARGO_BUILD_FEATURES=default +ARG RUST_RELEASE_MODE=debug + +WORKDIR /home/lemmy/src +USER 10001:10001 + +COPY --chown=lemmy:lemmy . ./ + +ENV RUST_RELEASE_MODE=${RUST_RELEASE_MODE} \ + CARGO_BUILD_FEATURES=${CARGO_BUILD_FEATURES} + +RUN --mount=type=cache,target=./target,uid=10001,gid=10001 bash ./docker/builders/lemmy-builder-arm64/docker-build.sh diff --git a/docker/builders/lemmy-builder-arm64/docker-build.sh b/docker/builders/lemmy-builder-arm64/docker-build.sh new file mode 100644 index 000000000..752fda80b --- /dev/null +++ b/docker/builders/lemmy-builder-arm64/docker-build.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -e; + +source "$HOME/.cargo/env" + +case "$RUST_RELEASE_MODE" in + "debug") + echo "pub const VERSION: &str = \"$(git describe --tag)\";" > "crates/utils/src/version.rs" + cargo build --features "${CARGO_BUILD_FEATURES}" + cp "./target/$CARGO_BUILD_TARGET/$RUST_RELEASE_MODE/lemmy_server" /home/lemmy/lemmy_server + ;; + "release") + # Pass a value to $USE_RELEASE_CACHE to avoid purging the cache for release builds + [[ -z "$USE_RELEASE_CACHE" ]] || cargo clean --release + echo "pub const VERSION: &str = \"$(git describe --tag)\";" > "crates/utils/src/version.rs" + cargo build --features "${CARGO_BUILD_FEATURES}" --release + cp "./target/$CARGO_BUILD_TARGET/$RUST_RELEASE_MODE/lemmy_server" /home/lemmy/lemmy_server + ;; +esac