diff --git a/RELEASING.md b/RELEASING.md index 25e031b2..8b04d453 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -9,15 +9,19 @@ 7. Copy release notes from [CHANGELOG.md](CHANGELOG.md) into [Github release](https://github.com/internetarchive/heritrix3/releases) 8. Attach {dist,contrib}/target/*-dist{.tar.gz,.zip} to Github release. ```bash - version=$(git describe --tags --abbrev=0) + version=$(git describe --tags --abbrev=0) && gh release upload $version target/checkout/dist/target/heritrix-$version-dist.zip target/checkout/dist/target/heritrix-$version-dist.tar.gz target/checkout/contrib/target/heritrix-contrib-$version-dist.tar.gz ``` -9. Build docker images: +9. Build docker images (from the dist tarballs uploaded to the Github release in the previous step): ```bash - version=$(git describe --tags --abbrev=0) - podman manifest create iipc/heritrix:$version - podman build --build-arg version=$version --platform linux/amd64,linux/arm64 --manifest iipc/heritrix:$version docker - podman manifest push --all iipc/heritrix:$version - podman manifest push --all iipc/heritrix:$version iipc/heritrix:latest + version=$(git describe --tags --abbrev=0) && + podman manifest create iipc/heritrix:$version && + podman build --build-arg version=$version --platform linux/amd64,linux/arm64 --manifest iipc/heritrix:$version docker && + podman manifest push --all iipc/heritrix:$version && + podman manifest push --all iipc/heritrix:$version iipc/heritrix:latest && + podman manifest create iipc/heritrix:$version-contrib && + podman build --build-arg version=$version --target contrib --platform linux/amd64,linux/arm64 --manifest iipc/heritrix:$version-contrib docker && + podman manifest push --all iipc/heritrix:$version-contrib && + podman manifest push --all iipc/heritrix:$version-contrib iipc/heritrix:contrib ``` 10. Announce in #heritrix (IIPC Slack) \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index 1063e8cf..ad42e941 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,53 +1,45 @@ -ARG java=25 +ARG java=25-jre-noble +ARG version=3.19.0 -FROM eclipse-temurin:${java} +FROM eclipse-temurin:${java} AS fetch-heritrix +ARG version +ADD https://github.com/internetarchive/heritrix3/releases/download/${version}/heritrix-${version}-dist.tar.gz /tmp/dist.tar.gz +RUN mkdir /opt/heritrix && tar -xzf /tmp/dist.tar.gz -C /opt/heritrix --strip-components=1 -ARG version="3.16.0" -ARG user="heritrix" +FROM eclipse-temurin:${java} AS fetch-contrib +ARG version +ARG TARGETARCH +ADD https://github.com/internetarchive/heritrix3/releases/download/${version}/heritrix-contrib-${version}-dist.tar.gz /tmp/dist.tar.gz +RUN mkdir /opt/heritrix && tar -xzf /tmp/dist.tar.gz -C /opt/heritrix --strip-components=1 +# yt-dlp for ExtractorYoutubeDL +ADD --chmod=755 https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux /tmp/yt-dlp-amd64 +ADD --chmod=755 https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux_aarch64 /tmp/yt-dlp-arm64 +RUN mv /tmp/yt-dlp-${TARGETARCH} /yt-dlp + +FROM eclipse-temurin:${java} AS base +ARG version ARG userid=1001 - -LABEL version=${version} -LABEL user=${user}/$userid - -# create user -RUN \ - groupadd -g $userid $user && \ - useradd -r -u $userid -g $user $user - -# Install dependencies -RUN \ - apt-get update && \ - apt-get install -y \ - wget \ - unzip && \ - rm -rf /var/lib/apt/lists/* - -WORKDIR /opt - -# download latest version according to: -# https://github.com/internetarchive/heritrix3/releases/tag/3.10.0 -RUN \ - wget -O heritrix-${version}-dist.zip https://github.com/internetarchive/heritrix3/releases/download/${version}/heritrix-${version}-dist.zip && \ - unzip heritrix-${version}-dist.zip && \ - rm heritrix-${version}-dist.zip && \ - mv heritrix-${version} heritrix && \ - chmod u+x heritrix/bin/heritrix && \ - chown -R $user:$user /opt/heritrix - -COPY entrypoint.sh /opt/entrypoint.sh -RUN chmod +x /opt/entrypoint.sh && \ - chown $user:$user /opt/entrypoint.sh - -WORKDIR /opt/heritrix - -USER $user - -ENV HERITRIX_HOME /opt/heritrix -# let it run in the foreground, required for docker -ENV FOREGROUND true - -# standard webport -# NOTE: that the webpage is via HTTPS only available! +LABEL org.opencontainers.image.title="Heritrix" \ + org.opencontainers.image.version="${version}" \ + org.opencontainers.image.source="https://github.com/internetarchive/heritrix3" +RUN groupadd --gid ${userid} heritrix && \ + useradd --system --uid ${userid} --gid heritrix heritrix +COPY --chmod=755 entrypoint.sh /opt/entrypoint.sh +ENV HERITRIX_HOME=/opt/heritrix \ + FOREGROUND=true +# workaround for CVE-2026-114938, remove once the base image has a fixed JDK +ENV JAVA_TOOL_OPTIONS="-XX:-UseBASE64Intrinsics" +# web UI (HTTPS only) EXPOSE 8443 - CMD ["/opt/entrypoint.sh"] + +FROM base AS contrib +COPY --from=fetch-contrib /yt-dlp /usr/local/bin/yt-dlp +COPY --from=fetch-contrib --chown=heritrix:heritrix /opt/heritrix /opt/heritrix +WORKDIR /opt/heritrix +USER heritrix + +FROM base AS heritrix +COPY --from=fetch-heritrix --chown=heritrix:heritrix /opt/heritrix /opt/heritrix +WORKDIR /opt/heritrix +USER heritrix diff --git a/docker/Dockerfile.contrib b/docker/Dockerfile.contrib deleted file mode 100644 index b439c2a6..00000000 --- a/docker/Dockerfile.contrib +++ /dev/null @@ -1,54 +0,0 @@ -ARG java=25-jdk-slim - -FROM openjdk:${java} - -ARG version="3.9.0" -ARG user="heritrix" -ARG userid=1000 - -LABEL version=${version} -LABEL user=${user}/$userid - -# create user -RUN \ - groupadd -g $userid $user && \ - useradd -r -u $userid -g $user $user - -# install dependencies and other requirements (for contrib) -RUN \ - apt-get update && \ - apt-get install -y --no-install-recommends \ - youtube-dl \ - wget \ - tar && \ - rm -rf /var/lib/apt/lists/* - -WORKDIR /opt - -# download latest version according to: -# https://repo1.maven.org/maven2/org/archive/heritrix/heritrix-contrib -RUN \ - wget -O heritrix-contrib-${version}-dist.tar.gz https://repo1.maven.org/maven2/org/archive/heritrix/heritrix-contrib/${version}/heritrix-contrib-${version}-dist.tar.gz && \ - tar xvfz heritrix-contrib-${version}-dist.tar.gz && \ - rm heritrix-contrib-${version}-dist.tar.gz && \ - mv heritrix-contrib-${version} heritrix && \ - chmod u+x heritrix/bin/heritrix && \ - chown -R $user:$user /opt/heritrix - -ADD entrypoint.sh /opt/entrypoint.sh -RUN chmod +x /opt/entrypoint.sh && \ - chown $user:$user /opt/entrypoint.sh - -WORKDIR /opt/heritrix - -USER $user - -ENV HERITRIX_HOME /opt/heritrix -# let it run in the foreground, required for docker -ENV FOREGROUND true - -# standard webport -# NOTE: that the webpage is via HTTPS only available! -EXPOSE 8443 - -CMD ["/opt/entrypoint.sh"] diff --git a/docker/Makefile b/docker/Makefile index 31d69d9a..0a4b9b4c 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -1,10 +1,10 @@ # makefile to build and publish images # You can override version and repo -# make image version=3.4.0-20210923 repo=iipc/ +# make image version=3.19.0 repo=iipc/ -.PHONY: all image-all-version run run-contrib image-all image image-contrib publish-all publish publish-contrib clean-all clean clean-contrib +.PHONY: all run run-contrib image-all image image-contrib publish-all publish publish-contrib clean-all clean clean-contrib -version ?= 3.4.0-20210923 +version ?= 3.19.0 repo = #pwd ?= $(shell pwd) @@ -23,7 +23,7 @@ image: docker tag heritrix:$(version) $(repo)heritrix:$(version)-jre image-contrib: - docker build --build-arg version=$(version) --build-arg java=8-jre -t heritrix:$(version)-contrib -f Dockerfile.contrib . + docker build --build-arg version=$(version) --target contrib -t heritrix:$(version)-contrib . docker tag heritrix:$(version)-contrib $(repo)heritrix:contrib docker tag heritrix:$(version)-contrib $(repo)heritrix:$(version)-contrib docker tag heritrix:$(version)-contrib $(repo)heritrix:$(version)-contrib-jre @@ -56,21 +56,6 @@ clean-contrib: # ------------------------------------ -image-all-version: - @echo "following (lower) version do not support the -r switch" - $(MAKE) image version=3.4.0-20200518 repo=$(repo) - $(MAKE) image version=3.4.0-20210527 repo=$(repo) - $(MAKE) image version=3.4.0-20210617 repo=$(repo) - $(MAKE) image version=3.4.0-20210618 repo=$(repo) - $(MAKE) image version=3.4.0-20210621 repo=$(repo) - @echo "following ones support the -r switch (run single job cli option)" - $(MAKE) image version=3.4.0-20210803 repo=$(repo) - # current ones - $(MAKE) image-contrib version=3.4.0-20210923 repo=$(repo) - $(MAKE) image version=3.4.0-20210923 repo=$(repo) - -# ------------------------------------ - run: [ -d jobs ] || mkdir jobs docker run --rm -it --name heritrix_container -p 8443:8443 -e "USERNAME=admin" -e "PASSWORD=admin" -v $$(pwd)/jobs:/opt/heritrix/jobs $(repo)heritrix:$(version)