📦 Reduce image sizes (dedupe)

FCP-INDI · Jul 18, 2023 · d5679f2 · d5679f2
1 parent fadfd3a
commit d5679f2
Show file tree

Hide file tree

Showing 16 changed files with 186 additions and 123 deletions.
diff --git a/.github/Dockerfiles/AFNI.23.1.10-jammy.Dockerfile b/.github/Dockerfiles/AFNI.23.1.10-jammy.Dockerfile
@@ -14,50 +14,104 @@
 
 # You should have received a copy of the GNU Lesser General Public
 # License along with C-PAC. If not, see <https://www.gnu.org/licenses/>.
+FROM ghcr.io/fcp-indi/c-pac/fsl:6.0.6.5-jammy as FSL
 FROM ghcr.io/fcp-indi/c-pac/ubuntu:jammy-non-free as AFNI
 USER root
+# To use the same Python environment to share common libraries
+COPY --from=FSL /usr/share/fsl/6.0 /usr/share/fsl/6.0
+ENV FSLDIR=/usr/share/fsl/6.0 \
+    PATH=/usr/share/fsl/6.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
 
 # install AFNI
 COPY dev/docker_data/required_afni_pkgs.txt /opt/required_afni_pkgs.txt
 COPY dev/docker_data/checksum/AFNI.23.1.10.sha384 /tmp/AFNI.23.1.10.sha384
 RUN apt-get update \
     && apt-get install -y \
+      apt-transport-https \
+      apt-utils \
+      bc \
       build-essential \
+      bzip2 \
+      ca-certificates \
       cmake \
       curl \
+      dh-autoreconf \
       eog \
       evince \
       firefox \
       gedit \
+      git \
       gnome-terminal \
       gnome-tweaks \
+      gnupg \
+      graphviz \
+      graphviz-dev \
       gsl-bin \
+      libcanberra-gtk-module \
       libcurl4-openssl-dev \
+      libexpat1-dev \
       libgdal-dev \
       libgfortran-11-dev \
+      libgiftiio-dev \
+      libgl1-mesa-dri \
+      libglib2.0-dev \
+      libglu1-mesa \
       libglu1-mesa-dev \
+      libglw1-mesa \
       libglw1-mesa-dev \
       libgomp1 \
+      libgsl-dev \
+      libjpeg-progs \
       libjpeg62 \
+      libmotif-dev \
       libnode-dev \
       libopenblas-dev \
       libssl-dev \
+      libtool \
       libudunits2-dev \
+      libx11-dev \
+      libxext-dev \
+      libxft-dev \
+      libxft2 \
+      libxi-dev \
       libxm4 \
+      libxml2 \
       libxml2-dev \
+      libxmu-dev \
+      libxmu-headers \
+      libxpm-dev \
+      libxslt1-dev \
+      locales \
+      m4 \
+      make \
+      mesa-common-dev \
+      mesa-utils \
       nautilus \
       netpbm \
-      python-is-python3 \
-      python3-matplotlib \
-      python3-numpy \
-      python3-pil \
+      ninja-build \
+      openssh-client \
+      pkg-config \
       r-base-dev \
+      rsync \
+      software-properties-common \
       tcsh \
+      unzip \
       vim \
+      wget \
+      x11proto-print-dev \
+      x11proto-xext-dev \
+      xauth \
       xfonts-100dpi \
       xfonts-base \
       xterm \
+      xutils-dev \
       xvfb \
+      zlib1g-dev && \
+    ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \
+    echo $TZ > /etc/timezone && \
+    sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \
+    dpkg-reconfigure --frontend=noninteractive locales && \
+    update-locale LANG="en_US.UTF-8" \
     && ln -s /usr/lib/x86_64-linux-gnu/libgsl.so.27 /usr/lib/x86_64-linux-gnu/libgsl.so.19 \
     && AFNI_VERSION="23.1.10" \
     && curl -LOJ https://github.com/afni/afni/archive/AFNI_${AFNI_VERSION}.tar.gz \
@@ -77,8 +131,33 @@ RUN apt-get update \
     ls > full_ls \
     && sed 's/linux_openmp_64\///g' /opt/required_afni_pkgs.txt | sort > required_ls \
     && comm -2 -3 full_ls required_ls | xargs rm -rf full_ls required_ls \
-    && apt-get remove -y libglw1-mesa-dev \
-    && ldconfig
+    # get rid of stuff we just needed for building
+    && apt-get remove -y \
+      apt-transport-https \
+      apt-utils \
+      build-essential \
+      bzip2 \
+      ca-certificates \
+      cmake \
+      curl \
+      dh-autoreconf \
+      evince \
+      firefox \
+      gedit \
+      git \
+      gnome-terminal \
+      gnome-tweaks \
+      libglw1-mesa-dev \
+      m4 \
+      make \
+      ninja-build \
+      openssh-client \
+      pkg-config \
+      unzip \
+      wget \
+      xterm \
+    && ldconfig \
+    && rm -rf /opt/afni/src
 
 # set up AFNI
 ENV PATH=/opt/afni:$PATH
@@ -90,7 +169,7 @@ RUN ldconfig
 
 RUN apt-get clean \
     && apt-get autoremove -y \
-    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/.cache/*
 
 FROM scratch
 LABEL org.opencontainers.image.description "NOT INTENDED FOR USE OTHER THAN AS A STAGE IMAGE IN A MULTI-STAGE BUILD \

diff --git a/.github/Dockerfiles/C-PAC.develop-jammy.Dockerfile b/.github/Dockerfiles/C-PAC.develop-jammy.Dockerfile
@@ -17,7 +17,13 @@ RUN rm -Rf /code/docker_data/Dockerfiles && \
 ENTRYPOINT ["/code/run-with-freesurfer.sh"]
 
 # link libraries & clean up
-RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/.cache/pip/* \
+# link libraries & clean up
+RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/.cache/* \
+    && find / -type f -print0 | sort -t/ -k2 | xargs -0 rdfind -makehardlinks true \
+    && rm -rf results.txt \
+    && apt-get remove rdfind -y \
+    && apt-get clean \
+    && apt-get autoremove -y \
     && ldconfig \
     && chmod 777 / \
     && chmod 777 $(ls / | grep -v sys | grep -v proc)

diff --git a/.github/Dockerfiles/C-PAC.develop-lite-jammy.Dockerfile b/.github/Dockerfiles/C-PAC.develop-lite-jammy.Dockerfile
@@ -33,7 +33,13 @@ RUN rm -Rf /code/docker_data/Dockerfiles && \
 ENTRYPOINT ["/code/run.py"]
 
 # link libraries & clean up
-RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/.cache/pip/* \
+# link libraries & clean up
+RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/.cache/* \
+    && find / -type f -print0 | sort -t/ -k2 | xargs -0 rdfind -makehardlinks true \
+    && rm -rf results.txt \
+    && apt-get remove rdfind -y \
+    && apt-get clean \
+    && apt-get autoremove -y \
     && ldconfig \
     && chmod 777 / \
     && chmod 777 $(ls / | grep -v sys | grep -v proc)

diff --git a/.github/Dockerfiles/FSL.6.0.6.5-jammy.Dockerfile b/.github/Dockerfiles/FSL.6.0.6.5-jammy.Dockerfile
@@ -16,8 +16,8 @@
 # License along with C-PAC. If not, see <https://www.gnu.org/licenses/>.
 FROM ghcr.io/fcp-indi/c-pac/fsl:data as data
 FROM ghcr.io/fcp-indi/c-pac/ubuntu:jammy-non-free AS FSL
-
 USER root
+ARG DEBIAN_FRONTEND=noninteractive
 COPY ./dev/docker_data/fsl/6.0.6.5 /tmp/fsl/manifest
 
 # set up FSL environment
@@ -35,10 +35,13 @@ ENV FSLOUTPUTTYPE=NIFTI_GZ \
 # Full list of packages, included excluded packages, in dev/docker_data/fsl/6.0.6.5/fsl-release.yml
 RUN apt-get update > /dev/null && \
     apt-get install --no-install-recommends --yes \
-        wget bzip2 ca-certificates \
-        git \
-        tini \
-        > /dev/null && \
+      bzip2 \
+      ca-certificates \
+      git \
+      python3 \
+      tini \
+      wget \
+      > /dev/null && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* && \
     wget --no-hsts --quiet https://github.com/conda-forge/miniforge/releases/download/23.1.0-4/Mambaforge-23.1.0-4-Linux-x86_64.sh -O /tmp/mambaforge.sh && \
@@ -82,22 +85,32 @@ RUN apt-get update > /dev/null && \
       --exclude_package '*xt*' \
       --manifest '/tmp/fsl/manifest/manifest.json' \
       --miniconda '/tmp/mambaforge.sh' \
-    && cd $FSLDIR \
-    && conda remove gcc make \
-    && ldconfig
+    # remove packages we just needed for building
+    && yes | mamba remove -n base \
+      c-compiler \
+      cxx-compiler \
+      fmrib-unpack \
+      gcc \
+      make \
+      nidmresults \
+      openslide-python \
+      vtk \
+    && ldconfig \
+    && yes | mamba clean --all \
+    && rm -rf /usr/share/fsl/6.0/pkgs/cache/*
 
 ENTRYPOINT ["/bin/bash"]
 
 # # Only keep what we need
-# FROM scratch
-# LABEL org.opencontainers.image.description "NOT INTENDED FOR USE OTHER THAN AS A STAGE IMAGE IN A MULTI-STAGE BUILD \
-# FSL 6.0.6.5 stage"
-# LABEL org.opencontainers.image.source https://github.com/FCP-INDI/C-PAC
-# COPY --from=FSL /lib/x86_64-linux-gnu /lib/x86_64-linux-gnu
-# COPY --from=FSL /usr/lib/x86_64-linux-gnu /usr/lib/x86_64-linux-gnu
-# COPY --from=FSL /usr/bin /usr/bin
-# COPY --from=FSL /usr/local/bin /usr/local/bin
-# COPY --from=FSL /usr/share/fsl /usr/share/fsl
+FROM scratch
+LABEL org.opencontainers.image.description "NOT INTENDED FOR USE OTHER THAN AS A STAGE IMAGE IN A MULTI-STAGE BUILD \
+FSL 6.0.6.5 stage"
+LABEL org.opencontainers.image.source https://github.com/FCP-INDI/C-PAC
+COPY --from=FSL /lib/x86_64-linux-gnu /lib/x86_64-linux-gnu
+COPY --from=FSL /usr/lib/x86_64-linux-gnu /usr/lib/x86_64-linux-gnu
+COPY --from=FSL /usr/bin /usr/bin
+COPY --from=FSL /usr/local/bin /usr/local/bin
+COPY --from=FSL /usr/share/fsl /usr/share/fsl
 # install C-PAC resources into FSL
 COPY --from=data /fsl_data/standard /usr/share/fsl/6.0/data/standard
 COPY --from=data /fsl_data/atlases /usr/share/fsl/6.0/data/atlases
diff --git a/.github/Dockerfiles/Ubuntu.jammy-non-free.Dockerfile b/.github/Dockerfiles/Ubuntu.jammy-non-free.Dockerfile
@@ -39,82 +39,19 @@ RUN groupadd -r c-pac \
     && chown -R c-pac_user:c-pac /home/c-pac_user \
     && chmod 777 / \
     && chmod ugo+w /etc/passwd \
-    # install general dependencies
+    && chmod 777 /opt \
+    && chmod a+s /opt \
+    # set up for noninteractive apt
     && apt-get update \
     && apt-get install -y --no-install-recommends \
       apt-utils \
-      apt-transport-https \
-      bc \
-      build-essential \
-      bzip2 \
-      ca-certificates \
-      cmake \
-      curl \
-      dc \
-      dh-autoreconf \
-      git \
-      gnupg \
-      graphviz \
-      graphviz-dev \
-      gsl-bin \
-      libcanberra-gtk-module \
-      libexpat1-dev \
-      libgiftiio-dev \
-      libglib2.0-dev \
-      libglu1-mesa \
-      libglu1-mesa-dev \
-      libjpeg-progs \
-      libgl1-mesa-dri \
-      libglw1-mesa \
-      libgsl-dev \
-      libmotif-dev \
-      libtool \
-      libx11-dev \
-      libxext-dev \
-      libxft2 \
-      libxft-dev \
-      libxi-dev \
-      libxml2 \
-      libxml2-dev \
-      libxmu-dev \
-      libxmu-headers \
-      libxpm-dev \
-      libxslt1-dev \
       locales \
-      m4 \
-      make \
-      mesa-common-dev \
-      mesa-utils \
-      netpbm \
-      ninja-build \
-      openssh-client \
-      pkg-config \
-      python-is-python3 \
-      python3 \
-      python3-dev \
-      python3-pip \
-      rsync \
-      software-properties-common \
-      tcsh \
-      unzip \
-      vim \
-      wget \
-      x11proto-dev \
-      xauth \
-      xutils-dev \
-      xvfb \
-      zlib1g-dev \
-  && ln -snf /usr/share/zoneinfo/$TZ /etc/localtime \
-  && echo $TZ > /etc/timezone \
-  && sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen \
-  && dpkg-reconfigure --frontend=noninteractive locales \
-  && update-locale LANG="en_US.UTF-8" \
-  && chmod 777 /opt \
-  && chmod a+s /opt
-
-# install Python dependencies
-COPY requirements.txt /opt/requirements.txt
-RUN pip install -r /opt/requirements.txt
+      rdfind \
+    && ln -snf /usr/share/zoneinfo/$TZ /etc/localtime \
+    && echo $TZ > /etc/timezone \
+    && sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen \
+    && dpkg-reconfigure --frontend=noninteractive locales \
+    && update-locale LANG="en_US.UTF-8"
 
 COPY --from=c-pac_templates /cpac_templates /cpac_templates
 COPY --from=dcan-hcp /opt/dcan-tools/pipeline/global /opt/dcan-tools/pipeline/global
@@ -127,10 +64,10 @@ COPY --from=c-pac_templates /opt/dcan-tools/pipeline/global/templates/Greyordina
 ENTRYPOINT ["/bin/bash"]
 
 # Link libraries for Singularity images
-RUN ldconfig && \
-    apt-get clean && \
-    apt-get autoremove -y && \
-    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/.cache/pip/*
+RUN ldconfig \
+    && apt-get clean \
+    && apt-get autoremove -y \
+    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/.cache/pip/*
 
 # Set user
 USER c-pac_user
diff --git a/.github/Dockerfiles/base-ABCD-HCP.Dockerfile b/.github/Dockerfiles/base-ABCD-HCP.Dockerfile
@@ -19,7 +19,7 @@ RUN chmod ugo+w /etc/passwd
 
 # install and set up c3d
 COPY --from=c3d /opt/c3d/ /opt/c3d/
-ENV C3DPATH=/opt/c3d/
+ENV C3DPATH=/opt/c3d
 ENV PATH=$C3DPATH/bin:$PATH
 
 # install AFNI