fedi/lemmy
1
0
Fork 0
mirror of https://github.com/LemmyNet/lemmy.git synced 2024-11-22 06:36:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Moving to debian-based docker image. Fixes #3972 (#4004)

Browse source 
* Moving to debian-based docker image. Fixes #3972

* Fix cargo.toml format.

* Try nightly rust for cargo_fmt.

* Trying to fix cargo build.

* Trying to fix cargo build 2.

* Trying to fix federation tests.

* Trying to fix federation tests 2.

* Trying to fix federation tests 3.

* Try to fix aarch 1.

* Try to fix aarch 2.

* Try to fix aarch 3.

* Removing aarch

* Adding tests back in.

* Adding ca-certs, and debian non-root user

* Commenting arm image building.

* Fix platform on runner.

* Version 0.19.0-beta.6
This commit is contained in:
Dessalines 2023-10-11 11:57:05 -04:00 committed by GitHub
parent 3a1be8c041
commit 645bf21d54
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 91 additions and 103 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
.woodpecker.yml
View file

@ -2,7 +2,7 @@
# See https://github.com/woodpecker-ci/woodpecker/issues/1677 # See https://github.com/woodpecker-ci/woodpecker/issues/1677
variables: variables:
- &muslrust_image "clux/muslrust:1.70.0" - &rust_image "rust:1.72.1"
- &slow_check_paths - &slow_check_paths
- path: - path:
# rust source code # rust source code
@ -57,15 +57,13 @@ steps:
cargo_fmt: cargo_fmt:
group: format group: format
image: *muslrust_image image: rustlang/rust:nightly
environment: environment:
# store cargo data in repo folder so that it gets cached between steps # store cargo data in repo folder so that it gets cached between steps
CARGO_HOME: .cargo CARGO_HOME: .cargo
commands: commands:
# need make existing toolchain available # need make existing toolchain available
- cp -n ~/.cargo . -r - cargo +nightly fmt -- --check
- rustup toolchain install nightly-2023-07-10 --no-self-update --profile minimal --component rustfmt
- cargo +nightly-2023-07-10 fmt -- --check
restore-cache: restore-cache:
image: meltwater/drone-cache:v1 image: meltwater/drone-cache:v1
@ -93,7 +91,7 @@ steps:
# make sure api builds with default features (used by other crates relying on lemmy api) # make sure api builds with default features (used by other crates relying on lemmy api)
check_api_common_default_features: check_api_common_default_features:
image: *muslrust_image image: *rust_image
environment: environment:
CARGO_HOME: .cargo CARGO_HOME: .cargo
commands: commands:
@ -101,7 +99,7 @@ steps:
when: *slow_check_paths when: *slow_check_paths
lemmy_api_common_doesnt_depend_on_diesel: lemmy_api_common_doesnt_depend_on_diesel:
image: *muslrust_image image: *rust_image
environment: environment:
CARGO_HOME: .cargo CARGO_HOME: .cargo
commands: commands:
@ -109,7 +107,7 @@ steps:
when: *slow_check_paths when: *slow_check_paths
lemmy_api_common_works_with_wasm: lemmy_api_common_works_with_wasm:
image: *muslrust_image image: *rust_image
environment: environment:
CARGO_HOME: .cargo CARGO_HOME: .cargo
commands: commands:
@ -118,7 +116,7 @@ steps:
when: *slow_check_paths when: *slow_check_paths
check_defaults_hjson_updated: check_defaults_hjson_updated:
image: *muslrust_image image: *rust_image
environment: environment:
CARGO_HOME: .cargo CARGO_HOME: .cargo
commands: commands:
@ -149,7 +147,7 @@ steps:
when: *slow_check_paths when: *slow_check_paths
cargo_clippy: cargo_clippy:
image: *muslrust_image image: *rust_image
environment: environment:
CARGO_HOME: .cargo CARGO_HOME: .cargo
commands: commands:
@ -173,17 +171,17 @@ steps:
when: *slow_check_paths when: *slow_check_paths
cargo_build: cargo_build:
image: *muslrust_image image: *rust_image
environment: environment:
CARGO_HOME: .cargo CARGO_HOME: .cargo
commands: commands:
- cargo build - cargo build
- mv target/x86_64-unknown-linux-musl/debug/lemmy_server target/lemmy_server - mv target/debug/lemmy_server target/lemmy_server
when: *slow_check_paths when: *slow_check_paths
cargo_test: cargo_test:
group: tests group: tests
image: *muslrust_image image: *rust_image
environment: environment:
LEMMY_DATABASE_URL: postgres://lemmy:password@database:5432/lemmy LEMMY_DATABASE_URL: postgres://lemmy:password@database:5432/lemmy
RUST_BACKTRACE: "1" RUST_BACKTRACE: "1"
@ -195,12 +193,12 @@ steps:
run_federation_tests: run_federation_tests:
group: tests group: tests
image: node:alpine image: node:20-bookworm-slim
environment: environment:
LEMMY_DATABASE_URL: postgres://lemmy:password@database:5432 LEMMY_DATABASE_URL: postgres://lemmy:password@database:5432
DO_WRITE_HOSTS_FILE: "1" DO_WRITE_HOSTS_FILE: "1"
commands: commands:
- apk add bash curl postgresql-client - apt update && apt install -y bash curl postgresql-client
- bash api_tests/prepare-drone-federation-test.sh - bash api_tests/prepare-drone-federation-test.sh
- cd api_tests/ - cd api_tests/
- yarn - yarn
@ -239,7 +237,9 @@ steps:
settings: settings:
repo: dessalines/lemmy repo: dessalines/lemmy
dockerfile: docker/Dockerfile dockerfile: docker/Dockerfile
platforms: linux/amd64,linux/arm64 # TODO fix arm build: see: https://woodpecker.join-lemmy.org/repos/129/pipeline/2888/20
# platforms: linux/amd64,linux/arm64
platforms: linux/amd64
build_args: build_args:
- RUST_RELEASE_MODE=release - RUST_RELEASE_MODE=release
tag: ${CI_COMMIT_TAG} tag: ${CI_COMMIT_TAG}

2
Cargo.toml
View file

@ -23,6 +23,8 @@ doctest = false
[profile.release] [profile.release]
debug = 0 debug = 0
lto = "thin" lto = "thin"
strip = true # Automatically strip symbols from the binary.
opt-level = "z" # Optimize for size.
# This profile significantly speeds up build time. If debug info is needed you can comment the line # This profile significantly speeds up build time. If debug info is needed you can comment the line
# out temporarily, but make sure to leave this in the main branch. # out temporarily, but make sure to leave this in the main branch.

160
docker/Dockerfile
View file

@ -1,54 +1,13 @@
# ARG RUST_VERSION=1.72.1
# Docker multiarch image:
# We build the Lemmy binary for amd64 and arm64 in individual stages using the blackdex/rust-musl image (github.com/blackdex/rust-musl).
# This image uses musl-cross-make (github.com/richfelker/musl-cross-make) to build a musl cross compilation toolchain for the target
# architecture. It also includes pre-built static libraries such as libpq. These libraries can improve the compile time and eliminate
# the requirement for extra dependencies in the final image.
#
# During each build stage, we use the blackdex/rust-musl openssl 3 images and configure PQ_LIB_DIR=/usr/local/musl/pq15/lib to use
# libpq v15. We also ensure the installation of the Rust toolchain corresponding to the target architecture using:
# `rustup target add $TARGET-unknown-linux-musl`.
#
ARG RUST_VERSION=1.71.0
ARG ALPINE_VERSION=3.18
ARG CARGO_BUILD_FEATURES=default ARG CARGO_BUILD_FEATURES=default
ARG RUST_RELEASE_MODE=debug ARG RUST_RELEASE_MODE=debug
ARG UID=911 ARG AMD_BUILDER_IMAGE=rust:${RUST_VERSION}
ARG GID=911 ARG ARM_BUILDER_IMAGE=blackdex/rust-musl:aarch64-musl-stable-${RUST_VERSION}-openssl3
ARG AMD_RUNNER_IMAGE=debian:bookworm-slim
# AMD64 builder base ARG ARM_RUNNER_IMAGE=alpine:3.18
FROM --platform=${BUILDPLATFORM} blackdex/rust-musl:x86_64-musl-stable-${RUST_VERSION}-openssl3 AS base-amd64
ENV DEBIAN_FRONTEND=noninteractive
ENV CARGO_HOME=/root/.cargo
ENV PQ_LIB_DIR=/usr/local/musl/pq15/lib
RUN apt update && apt install -y \
--no-install-recommends \
git
RUN mkdir -pv "${CARGO_HOME}" && \
rustup set profile minimal && \
rustup target add x86_64-unknown-linux-musl
# ARM64 builder base
FROM --platform=${BUILDPLATFORM} blackdex/rust-musl:aarch64-musl-stable-${RUST_VERSION}-openssl3 AS base-arm64
ENV DEBIAN_FRONTEND=noninteractive
ENV CARGO_HOME=/root/.cargo
ENV PQ_LIB_DIR=/usr/local/musl/pq15/lib
RUN apt update && apt install -y \
--no-install-recommends \
git
RUN mkdir -pv "${CARGO_HOME}" && \
rustup set profile minimal && \
rustup target add aarch64-unknown-linux-musl
# AMD64 builder # AMD64 builder
FROM base-amd64 AS build-amd64 FROM --platform=${BUILDPLATFORM} ${AMD_BUILDER_IMAGE} AS build-amd64
ARG CARGO_BUILD_FEATURES ARG CARGO_BUILD_FEATURES
ARG RUST_RELEASE_MODE ARG RUST_RELEASE_MODE
@ -61,65 +20,92 @@ COPY . ./
RUN --mount=type=cache,target=/lemmy/target set -ex; \ RUN --mount=type=cache,target=/lemmy/target set -ex; \
if [ "${RUST_RELEASE_MODE}" = "debug" ]; then \ if [ "${RUST_RELEASE_MODE}" = "debug" ]; then \
echo "pub const VERSION: &str = \"$(git describe --tag)\";" > crates/utils/src/version.rs; \ echo "pub const VERSION: &str = \"$(git describe --tag)\";" > crates/utils/src/version.rs; \
cargo build --target=x86_64-unknown-linux-musl --features "${CARGO_BUILD_FEATURES}"; \ cargo build --features "${CARGO_BUILD_FEATURES}"; \
mv target/x86_64-unknown-linux-musl/debug/lemmy_server ./lemmy; \ mv target/debug/lemmy_server ./lemmy; \
fi fi
# Release build # Release build
RUN set -ex; \ RUN set -ex; \
if [ "${RUST_RELEASE_MODE}" = "release" ]; then \ if [ "${RUST_RELEASE_MODE}" = "release" ]; then \
echo "pub const VERSION: &str = \"$(git describe --tag)\";" > crates/utils/src/version.rs; \ echo "pub const VERSION: &str = \"$(git describe --tag)\";" > crates/utils/src/version.rs; \
cargo build --target=x86_64-unknown-linux-musl --features "${CARGO_BUILD_FEATURES}" --release; \ cargo build --features "${CARGO_BUILD_FEATURES}" --release; \
mv target/x86_64-unknown-linux-musl/release/lemmy_server ./lemmy; \ mv target/release/lemmy_server ./lemmy; \
fi fi
# ARM64 builder # ARM64 builder
FROM base-arm64 AS build-arm64 # TODO currently broken
# FROM --platform=${BUILDPLATFORM} ${ARM_BUILDER_IMAGE} as build-arm64
ARG CARGO_BUILD_FEATURES # ENV DEBIAN_FRONTEND=noninteractive
ARG RUST_RELEASE_MODE # ENV CARGO_HOME=/root/.cargo
# ENV PQ_LIB_DIR=/usr/local/musl/pq15/lib
WORKDIR /lemmy # RUN apt update && apt install -y \
# --no-install-recommends \
# git
COPY . ./ # RUN mkdir -pv "${CARGO_HOME}" && \
# rustup set profile minimal && \
# rustup target add aarch64-unknown-linux-musl
# Debug build # ARG CARGO_BUILD_FEATURES
RUN --mount=type=cache,target=/lemmy/target set -ex; \ # ARG RUST_RELEASE_MODE
if [ "${RUST_RELEASE_MODE}" = "debug" ]; then \
echo "pub const VERSION: &str = \"$(git describe --tag)\";" > crates/utils/src/version.rs; \
cargo build --target=aarch64-unknown-linux-musl --features "${CARGO_BUILD_FEATURES}"; \
mv target/aarch64-unknown-linux-musl/debug/lemmy_server ./lemmy; \
fi
# Release build # WORKDIR /lemmy
RUN set -ex; \
if [ "${RUST_RELEASE_MODE}" = "release" ]; then \
echo "pub const VERSION: &str = \"$(git describe --tag)\";" > crates/utils/src/version.rs; \
cargo build --target=aarch64-unknown-linux-musl --features "${CARGO_BUILD_FEATURES}" --release; \
mv target/aarch64-unknown-linux-musl/release/lemmy_server ./lemmy; \
fi
# Get target binary # COPY . ./
FROM build-${TARGETARCH} AS build
# # Debug build
# RUN --mount=type=cache,target=/lemmy/target set -ex; \
# if [ "${RUST_RELEASE_MODE}" = "debug" ]; then \
# echo "pub const VERSION: &str = \"$(git describe --tag)\";" > crates/utils/src/version.rs; \
# cargo build --target=aarch64-unknown-linux-musl --features "${CARGO_BUILD_FEATURES}"; \
# mv target/aarch64-unknown-linux-musl/debug/lemmy_server ./lemmy; \
# fi
# # Release build
# RUN set -ex; \
# if [ "${RUST_RELEASE_MODE}" = "release" ]; then \
# echo "pub const VERSION: &str = \"$(git describe --tag)\";" > crates/utils/src/version.rs; \
# cargo build --target=aarch64-unknown-linux-musl --features "${CARGO_BUILD_FEATURES}" --release; \
# mv target/aarch64-unknown-linux-musl/release/lemmy_server ./lemmy; \
# fi
## Final image ## Final image
FROM alpine:${ALPINE_VERSION} FROM ${AMD_RUNNER_IMAGE}
ARG UID # Federation needs CA certificates
ARG GID RUN apt update && apt install -y libssl-dev libpq-dev ca-certificates
RUN apk add --no-cache \ # Debian / Ubuntu non-root user creds
ca-certificates ARG UNAME=lemmy
ARG UID=1000
COPY --from=build --chmod=0755 /lemmy/lemmy /usr/local/bin ARG GID=1000
RUN groupadd -g $GID -o $UNAME
RUN addgroup -S -g ${GID} lemmy && \ RUN useradd -m -u $UID -g $GID -o -s /bin/bash $UNAME
adduser -S -H -D -G lemmy -u ${UID} -g "" -s /sbin/nologin lemmy USER $UNAME
USER lemmy
CMD ["lemmy"]
COPY --from=build-amd64 /lemmy/lemmy ./
CMD ["./lemmy"]
EXPOSE 8536 EXPOSE 8536
STOPSIGNAL SIGTERM STOPSIGNAL SIGTERM
## Arm Runner
# FROM --platform=${BUILDPLATFORM} ${ARM_RUNNER_IMAGE}
# ARG UNAME=lemmy
# ARG UID=1000
# ARG GID=1000
# RUN apk add --no-cache ca-certificates
# COPY --from=build-arm64 --chmod=0755 /lemmy/lemmy /usr/local/bin
# RUN addgroup -S -g ${GID} ${UNAME} && \
# adduser -S -H -D -G ${UNAME} -u ${UID} -g "" -s /sbin/nologin ${UNAME}
# USER $UNAME
# CMD ["lemmy"]
# EXPOSE 8536
# STOPSIGNAL SIGTERM