From cf5b0472203cd28d03288eee04950eb41e8f6bac Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Fri, 23 Jul 2021 15:45:09 +0300 Subject: [PATCH 1/6] [Security] Use ubuntu:20.04 base image for Pulsar docker images - Ubuntu fixes critical and high security vulnerabilities. - openjdk:11-jdk-slim/openjdk:11-jdk images are based on Debian 10 which contains a lot of unfixed vulnerabilities. - this causes the Pulsar docker images to get flagged in Docker image vulnerability scanning with docker image vulnerability scanning tools such as Clair --- docker/pulsar-standalone/Dockerfile | 9 ++++----- docker/pulsar/Dockerfile | 17 ++++++++--------- docker/pulsar/pom.xml | 4 ++-- ...ar-client-37.sh => install-pulsar-client.sh} | 5 +++-- tests/docker-images/java-test-image/Dockerfile | 4 ++-- 5 files changed, 19 insertions(+), 20 deletions(-) rename docker/pulsar/scripts/{install-pulsar-client-37.sh => install-pulsar-client.sh} (79%) diff --git a/docker/pulsar-standalone/Dockerfile b/docker/pulsar-standalone/Dockerfile index 777541b168872..01718e46a3dbe 100644 --- a/docker/pulsar-standalone/Dockerfile +++ b/docker/pulsar-standalone/Dockerfile @@ -24,19 +24,18 @@ FROM apachepulsar/pulsar-all:latest as pulsar FROM apachepulsar/pulsar-dashboard:latest as dashboard # Restart from -FROM openjdk:11-jdk +FROM ubuntu:20.04 # Note that the libpq-dev package is needed here in order to install # the required python psycopg2 package (for postgresql) later RUN apt-get update \ - && apt-get -y install python3.7 python3.7-dev python3-pip postgresql sudo nginx supervisor libpq-dev + && apt-get -y install openjdk-11-jdk-headless python3 python3-dev python3-pip postgresql sudo nginx supervisor libpq-dev RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 RUN update-ca-certificates -RUN ln -s /usr/bin/pip3 /usr/bin/pip # Postgres configuration -COPY --from=dashboard /etc/postgresql/11/main/postgresql.conf /etc/postgresql/11/main/postgresql.conf +COPY --from=dashboard /etc/postgresql/11/main/postgresql.conf /etc/postgresql/12/main/postgresql.conf # Configure supervisor COPY --from=dashboard /etc/supervisor/conf.d/supervisor-app.conf /etc/supervisor/conf.d/supervisor-app.conf @@ -73,7 +72,7 @@ RUN pip install -r /pulsar/django/requirements.txt COPY --from=dashboard /pulsar/init-postgres.sh /pulsar/django/init-postgres.sh RUN mkdir /data RUN /pulsar/django/init-postgres.sh -RUN sudo -u postgres /etc/init.d/postgresql stop +RUN sudo -u postgres /etc/init.d/postgresql stop # Add postgresql to supervisord. Redirect logs to stdout RUN echo "\n[program:postgresql]\n\ command = /etc/init.d/postgresql start\n\ diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile index 96da0d2822bc0..adf9b5c067a26 100644 --- a/docker/pulsar/Dockerfile +++ b/docker/pulsar/Dockerfile @@ -31,26 +31,26 @@ COPY scripts/gen-yml-from-env.py /pulsar/bin COPY scripts/generate-zookeeper-config.sh /pulsar/bin COPY scripts/pulsar-zookeeper-ruok.sh /pulsar/bin COPY scripts/watch-znode.py /pulsar/bin -COPY scripts/install-pulsar-client-37.sh /pulsar/bin +COPY scripts/install-pulsar-client.sh /pulsar/bin RUN mkdir /pulsar/data -### Create 2nd stage from OpenJDK image -### and add Python dependencies (for Pulsar functions) +### Create 2nd stage from Ubuntu image +### and add OpenJDK and Python dependencies (for Pulsar functions) -FROM openjdk:11-jdk-slim +FROM ubuntu:20.04 # Install some utilities RUN apt-get update \ - && apt-get install -y netcat dnsutils less procps iputils-ping \ - python3.7 python3.7-dev python3-setuptools python3-yaml python3-kazoo \ + && apt-get -y install openjdk-11-jdk-headless netcat dnsutils less procps iputils-ping \ + python3 python3-dev python3-setuptools python3-yaml python3-kazoo \ libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev \ curl \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py -RUN python3.7 get-pip.py +RUN python3 get-pip.py RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 @@ -65,5 +65,4 @@ ENV PULSAR_ROOT_LOGGER=INFO,CONSOLE COPY --from=pulsar /pulsar /pulsar WORKDIR /pulsar -RUN /pulsar/bin/install-pulsar-client-37.sh - +RUN /pulsar/bin/install-pulsar-client.sh \ No newline at end of file diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index 43d918faa4ec0..06e21e9d46b02 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -74,8 +74,8 @@ ${project.basedir}/target ${project.basedir}/../../pulsar-client-cpp/docker/build-wheels.sh - - 3.7 cp37-cp37m + + 3.8 cp38-cp38 diff --git a/docker/pulsar/scripts/install-pulsar-client-37.sh b/docker/pulsar/scripts/install-pulsar-client.sh similarity index 79% rename from docker/pulsar/scripts/install-pulsar-client-37.sh rename to docker/pulsar/scripts/install-pulsar-client.sh index 44932b6cb0ef6..56702b10db3d8 100755 --- a/docker/pulsar/scripts/install-pulsar-client-37.sh +++ b/docker/pulsar/scripts/install-pulsar-client.sh @@ -20,5 +20,6 @@ set -x -WHEEL_FILE=$(ls /pulsar/pulsar-client | grep cp37) -pip3.7 install /pulsar/pulsar-client/${WHEEL_FILE}[all] +PYTHON_MAJOR_MINOR=$(python3 -V | sed -E 's/.* ([[:digit:]]+)\.([[:digit:]]+).*/\1\2/') +WHEEL_FILE=$(ls /pulsar/pulsar-client | grep "cp${PYTHON_MAJOR_MINOR}") +pip3 install /pulsar/pulsar-client/${WHEEL_FILE}[all] diff --git a/tests/docker-images/java-test-image/Dockerfile b/tests/docker-images/java-test-image/Dockerfile index 708a524cbcdb5..6081c0156abdb 100644 --- a/tests/docker-images/java-test-image/Dockerfile +++ b/tests/docker-images/java-test-image/Dockerfile @@ -17,7 +17,7 @@ # under the License. # -FROM openjdk:11-jdk-slim +FROM ubuntu:20.04 RUN groupadd -g 10001 pulsar RUN adduser -u 10000 --gid 10001 --disabled-login --disabled-password --gecos '' pulsar @@ -34,7 +34,7 @@ RUN echo networkaddress.cache.ttl=1 >> $JAVA_HOME/conf/security/java.security WORKDIR /pulsar -RUN apt-get update +RUN apt-get update && apt-get -y install openjdk-11-jdk-headless # /pulsar/bin/watch-znode.py requires python3-kazoo # /pulsar/bin/pulsar-managed-ledger-admin requires python3-protobuf From b91b672918fc1d2b83c8b7820ebc14fa00d2ac76 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Fri, 23 Jul 2021 15:53:22 +0300 Subject: [PATCH 2/6] Install Ubuntu updates --- docker/pulsar/Dockerfile | 3 +++ tests/docker-images/java-test-image/Dockerfile | 8 ++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile index adf9b5c067a26..2b5ac0f67cb20 100644 --- a/docker/pulsar/Dockerfile +++ b/docker/pulsar/Dockerfile @@ -42,10 +42,13 @@ FROM ubuntu:20.04 # Install some utilities RUN apt-get update \ + && apt-get -y dist-upgrade \ && apt-get -y install openjdk-11-jdk-headless netcat dnsutils less procps iputils-ping \ python3 python3-dev python3-setuptools python3-yaml python3-kazoo \ libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev \ curl \ + && apt-get -y --purge autoremove \ + && apt-get autoclean \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/tests/docker-images/java-test-image/Dockerfile b/tests/docker-images/java-test-image/Dockerfile index 6081c0156abdb..ac219cde65420 100644 --- a/tests/docker-images/java-test-image/Dockerfile +++ b/tests/docker-images/java-test-image/Dockerfile @@ -34,7 +34,9 @@ RUN echo networkaddress.cache.ttl=1 >> $JAVA_HOME/conf/security/java.security WORKDIR /pulsar -RUN apt-get update && apt-get -y install openjdk-11-jdk-headless +RUN apt-get update \ + && apt-get -y dist-upgrade \ + && apt-get -y install openjdk-11-jdk-headless # /pulsar/bin/watch-znode.py requires python3-kazoo # /pulsar/bin/pulsar-managed-ledger-admin requires python3-protobuf @@ -64,5 +66,7 @@ ENV PULSAR_ROOT_LOGGER=INFO,CONSOLE RUN chown -R pulsar:0 /pulsar && chmod -R g=u /pulsar # cleanup -RUN apt-get clean \ +RUN apt-get -y --purge autoremove \ + && apt-get autoclean \ + && apt-get clean \ && rm -rf /var/lib/apt/lists/* \ No newline at end of file From 70e02ab6e24f8070f4e435dbea6283df95bacea3 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Fri, 23 Jul 2021 20:21:00 +0300 Subject: [PATCH 3/6] Set DEBIAN_FRONTEND=noninteractive so that dist-upgrade doesn't wait for input --- docker/pulsar/Dockerfile | 2 ++ tests/docker-images/java-test-image/Dockerfile | 2 ++ 2 files changed, 4 insertions(+) diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile index 2b5ac0f67cb20..c91272f60c56e 100644 --- a/docker/pulsar/Dockerfile +++ b/docker/pulsar/Dockerfile @@ -40,6 +40,8 @@ RUN mkdir /pulsar/data FROM ubuntu:20.04 +ARG DEBIAN_FRONTEND=noninteractive + # Install some utilities RUN apt-get update \ && apt-get -y dist-upgrade \ diff --git a/tests/docker-images/java-test-image/Dockerfile b/tests/docker-images/java-test-image/Dockerfile index ac219cde65420..c0758a7947b21 100644 --- a/tests/docker-images/java-test-image/Dockerfile +++ b/tests/docker-images/java-test-image/Dockerfile @@ -34,6 +34,8 @@ RUN echo networkaddress.cache.ttl=1 >> $JAVA_HOME/conf/security/java.security WORKDIR /pulsar +ARG DEBIAN_FRONTEND=noninteractive + RUN apt-get update \ && apt-get -y dist-upgrade \ && apt-get -y install openjdk-11-jdk-headless From 10d9a2414943d6414f4058881cec45a3cb7f693f Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Fri, 23 Jul 2021 20:50:12 +0300 Subject: [PATCH 4/6] Set JAVA_HOME --- docker/pulsar/Dockerfile | 1 + tests/docker-images/java-test-image/Dockerfile | 1 + 2 files changed, 2 insertions(+) diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile index c91272f60c56e..212f642f4a9ed 100644 --- a/docker/pulsar/Dockerfile +++ b/docker/pulsar/Dockerfile @@ -59,6 +59,7 @@ RUN python3 get-pip.py RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 +ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64 RUN echo networkaddress.cache.ttl=1 >> $JAVA_HOME/conf/security/java.security ADD target/python-client/ /pulsar/pulsar-client diff --git a/tests/docker-images/java-test-image/Dockerfile b/tests/docker-images/java-test-image/Dockerfile index c0758a7947b21..ca770d6685d6d 100644 --- a/tests/docker-images/java-test-image/Dockerfile +++ b/tests/docker-images/java-test-image/Dockerfile @@ -30,6 +30,7 @@ RUN chown -R root:root /pulsar COPY target/scripts /pulsar/bin RUN chmod a+rx /pulsar/bin/* +ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64 RUN echo networkaddress.cache.ttl=1 >> $JAVA_HOME/conf/security/java.security WORKDIR /pulsar From 7ff11f933b73053e7decc96b4063ac66634c1a8c Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Fri, 23 Jul 2021 21:34:05 +0300 Subject: [PATCH 5/6] Fix configuring networkaddress.cache.ttl - JAVA_HOME set by ENV isn't available for RUN commands --- docker/pulsar/Dockerfile | 2 +- tests/docker-images/java-test-image/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile index 212f642f4a9ed..d30c6c3b681a7 100644 --- a/docker/pulsar/Dockerfile +++ b/docker/pulsar/Dockerfile @@ -60,7 +60,7 @@ RUN python3 get-pip.py RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64 -RUN echo networkaddress.cache.ttl=1 >> $JAVA_HOME/conf/security/java.security +RUN echo networkaddress.cache.ttl=1 >> /usr/lib/jvm/java-11-openjdk-amd64/conf/security/java.security ADD target/python-client/ /pulsar/pulsar-client VOLUME ["/pulsar/conf", "/pulsar/data"] diff --git a/tests/docker-images/java-test-image/Dockerfile b/tests/docker-images/java-test-image/Dockerfile index ca770d6685d6d..91c0012b1868b 100644 --- a/tests/docker-images/java-test-image/Dockerfile +++ b/tests/docker-images/java-test-image/Dockerfile @@ -31,7 +31,7 @@ COPY target/scripts /pulsar/bin RUN chmod a+rx /pulsar/bin/* ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64 -RUN echo networkaddress.cache.ttl=1 >> $JAVA_HOME/conf/security/java.security +RUN echo networkaddress.cache.ttl=1 >> /usr/lib/jvm/java-11-openjdk-amd64/conf/security/java.security WORKDIR /pulsar From c90e212ae43f404e688812f8af72281c7d3a93f5 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Fri, 23 Jul 2021 22:05:33 +0300 Subject: [PATCH 6/6] Configure networkaddress.cache.ttl after installing OpenJDK --- tests/docker-images/java-test-image/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/docker-images/java-test-image/Dockerfile b/tests/docker-images/java-test-image/Dockerfile index 91c0012b1868b..2d5e75fbc71fe 100644 --- a/tests/docker-images/java-test-image/Dockerfile +++ b/tests/docker-images/java-test-image/Dockerfile @@ -30,9 +30,6 @@ RUN chown -R root:root /pulsar COPY target/scripts /pulsar/bin RUN chmod a+rx /pulsar/bin/* -ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64 -RUN echo networkaddress.cache.ttl=1 >> /usr/lib/jvm/java-11-openjdk-amd64/conf/security/java.security - WORKDIR /pulsar ARG DEBIAN_FRONTEND=noninteractive @@ -41,6 +38,9 @@ RUN apt-get update \ && apt-get -y dist-upgrade \ && apt-get -y install openjdk-11-jdk-headless +ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64 +RUN echo networkaddress.cache.ttl=1 >> /usr/lib/jvm/java-11-openjdk-amd64/conf/security/java.security + # /pulsar/bin/watch-znode.py requires python3-kazoo # /pulsar/bin/pulsar-managed-ledger-admin requires python3-protobuf RUN apt-get install -y python3-kazoo python3-protobuf