From 1001927d12b3c28902024ccda39b22b8880fde9c Mon Sep 17 00:00:00 2001 From: Mihai Maruseac Date: Wed, 3 Jun 2020 12:35:13 -0700 Subject: [PATCH 1/7] Use bazelisk instead of the grep configure -> get Bazel version -> curl hack --- projects/tensorflow/Dockerfile | 4 ++++ projects/tensorflow/build.sh | 25 ------------------------- 2 files changed, 4 insertions(+), 25 deletions(-) diff --git a/projects/tensorflow/Dockerfile b/projects/tensorflow/Dockerfile index 64100cf25947..1b19db13323f 100644 --- a/projects/tensorflow/Dockerfile +++ b/projects/tensorflow/Dockerfile @@ -30,6 +30,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list RUN curl https://bazel.build/bazel-release.pub.gpg | apt-key add - RUN apt-get update && apt-get install -y bazel +RUN curl -Lo /usr/bin/bazel \ + https://github.com/bazelbuild/bazelisk/releases/download/v1.1.0/bazelisk-linux-amd64 \ + && \ + chmod +x /usr/bin/bazel RUN git clone --depth 1 https://github.com/tensorflow/tensorflow tensorflow WORKDIR $SRC/tensorflow diff --git a/projects/tensorflow/build.sh b/projects/tensorflow/build.sh index b1589e0e8b02..1349be4c09bb 100755 --- a/projects/tensorflow/build.sh +++ b/projects/tensorflow/build.sh @@ -15,31 +15,6 @@ # ################################################################################ -# First, determine the latest Bazel we can support -BAZEL_VERSION=$( - grep '_TF_MAX_BAZEL_VERSION =' configure.py | \ - cut -d\' -f2 | tr -d '[:space:]' -) -if [ -z ${BAZEL_VERSION} ]; then - echo "Couldn't find a valid bazel version in configure.py script" - exit 1 -fi - -# Then, install it -curl -fSsL -O https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh -chmod +x ./bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh -./bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh - -# Finally, check instalation before proceeding to compile -INSTALLED_VERSION=$( - bazel version | grep 'Build label' | cut -d: -f2 | tr -d '[:space:]' -) -if [ ${INSTALLED_VERSION} != ${BAZEL_VERSION} ]; then - echo "Couldn't install required Bazel. " - echo "Want ${BAZEL_VERSION}. Got ${INSTALLED_VERSION}." - exit 1 -fi - # Generate the list of fuzzers we have (only the base/op name). FUZZING_BUILD_FILE="tensorflow/core/kernels/fuzzing/BUILD" declare -r FUZZERS=$( From 2d480bc0c7cc0a1736606b087c39ab6b0eb8646a Mon Sep 17 00:00:00 2001 From: Mihai Maruseac Date: Wed, 3 Jun 2020 12:34:26 -0700 Subject: [PATCH 2/7] Use python3 as python2 is deprecated --- projects/tensorflow/Dockerfile | 4 ++-- projects/tensorflow/build.sh | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/projects/tensorflow/Dockerfile b/projects/tensorflow/Dockerfile index 1b19db13323f..403d0a7869a7 100644 --- a/projects/tensorflow/Dockerfile +++ b/projects/tensorflow/Dockerfile @@ -19,8 +19,8 @@ MAINTAINER mihaimaruseac@google.com RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ - python-dev \ - python-future \ + python3-dev \ + python3-future \ rsync \ && \ apt-get clean && \ diff --git a/projects/tensorflow/build.sh b/projects/tensorflow/build.sh index 1349be4c09bb..d1e168a005c3 100755 --- a/projects/tensorflow/build.sh +++ b/projects/tensorflow/build.sh @@ -27,8 +27,11 @@ declare -r FUZZERS=$( CFLAGS="${CFLAGS} -fno-sanitize=vptr" CXXFLAGS="${CXXFLAGS} -fno-sanitize=vptr -std=c++11 -stdlib=libc++" +# Force Python3 and install required python deps +PYTHON=python3 + # Make sure we run ./configure to detect when we are using a Bazel out of range -yes "" | ./configure +yes "" | ${PYTHON} configure.py # See https://github.com/bazelbuild/bazel/issues/6697 sed '/::kM..SeedBytes/d' -i tensorflow/stream_executor/rng.cc From a63416e1546d264e877d78cced41962665579176 Mon Sep 17 00:00:00 2001 From: Mihai Maruseac Date: Wed, 3 Jun 2020 12:34:47 -0700 Subject: [PATCH 3/7] Need to manually install numpy now as it is used in the toolchain --- projects/tensorflow/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/tensorflow/build.sh b/projects/tensorflow/build.sh index d1e168a005c3..633ca6aa8c63 100755 --- a/projects/tensorflow/build.sh +++ b/projects/tensorflow/build.sh @@ -29,6 +29,7 @@ CXXFLAGS="${CXXFLAGS} -fno-sanitize=vptr -std=c++11 -stdlib=libc++" # Force Python3 and install required python deps PYTHON=python3 +${PYTHON} -m pip install numpy # Make sure we run ./configure to detect when we are using a Bazel out of range yes "" | ${PYTHON} configure.py From c8ba9131254546530458174e75437932093e3285 Mon Sep 17 00:00:00 2001 From: Mihai Maruseac Date: Wed, 3 Jun 2020 12:05:04 -0700 Subject: [PATCH 4/7] Remove dga@google.com as he left Google --- projects/tensorflow/project.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/projects/tensorflow/project.yaml b/projects/tensorflow/project.yaml index dfdd60953c78..57ff1aeafa31 100644 --- a/projects/tensorflow/project.yaml +++ b/projects/tensorflow/project.yaml @@ -2,7 +2,6 @@ homepage: "https://www.tensorflow.org" language: c++ primary_contact: "mihaimaruseac@google.com" auto_ccs: - - "dga@google.com" - "frankchn@google.com" fuzzing_engines: - libfuzzer From d29639714419f3de2cbf1715adfa579e9a0db9e6 Mon Sep 17 00:00:00 2001 From: Mihai Maruseac Date: Wed, 3 Jun 2020 13:32:44 -0700 Subject: [PATCH 5/7] Remove C++11 constraint as TF now builds and uses C++14 --- projects/tensorflow/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/tensorflow/build.sh b/projects/tensorflow/build.sh index 633ca6aa8c63..f109ac698eba 100755 --- a/projects/tensorflow/build.sh +++ b/projects/tensorflow/build.sh @@ -25,7 +25,7 @@ declare -r FUZZERS=$( # Note the c++11/libc++ flags to build using the same toolchain as the one used # to build libFuzzingEngine. CFLAGS="${CFLAGS} -fno-sanitize=vptr" -CXXFLAGS="${CXXFLAGS} -fno-sanitize=vptr -std=c++11 -stdlib=libc++" +CXXFLAGS="${CXXFLAGS} -fno-sanitize=vptr" # Force Python3 and install required python deps PYTHON=python3 From 9262b020525797744b063c5c4d8831f643cb1f73 Mon Sep 17 00:00:00 2001 From: Mihai Maruseac Date: Wed, 3 Jun 2020 14:51:05 -0700 Subject: [PATCH 6/7] Handle review --- projects/tensorflow/Dockerfile | 3 +-- projects/tensorflow/build.sh | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/projects/tensorflow/Dockerfile b/projects/tensorflow/Dockerfile index 403d0a7869a7..ab4bae1e3fce 100644 --- a/projects/tensorflow/Dockerfile +++ b/projects/tensorflow/Dockerfile @@ -19,12 +19,11 @@ MAINTAINER mihaimaruseac@google.com RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ - python3-dev \ - python3-future \ rsync \ && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* +RUN python3 -m pip install numpy # Install Bazel from apt-get to ensure dependencies are there RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list diff --git a/projects/tensorflow/build.sh b/projects/tensorflow/build.sh index f109ac698eba..38208b889326 100755 --- a/projects/tensorflow/build.sh +++ b/projects/tensorflow/build.sh @@ -29,7 +29,6 @@ CXXFLAGS="${CXXFLAGS} -fno-sanitize=vptr" # Force Python3 and install required python deps PYTHON=python3 -${PYTHON} -m pip install numpy # Make sure we run ./configure to detect when we are using a Bazel out of range yes "" | ${PYTHON} configure.py From 3ee2bcf8cdc22144a42155534ee28a4cd9cbfcf1 Mon Sep 17 00:00:00 2001 From: Mihai Maruseac Date: Wed, 3 Jun 2020 14:51:22 -0700 Subject: [PATCH 7/7] Remove source sed/replace as it is no longer needed --- projects/tensorflow/build.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/projects/tensorflow/build.sh b/projects/tensorflow/build.sh index 38208b889326..777821ecd4df 100755 --- a/projects/tensorflow/build.sh +++ b/projects/tensorflow/build.sh @@ -33,9 +33,6 @@ PYTHON=python3 # Make sure we run ./configure to detect when we are using a Bazel out of range yes "" | ${PYTHON} configure.py -# See https://github.com/bazelbuild/bazel/issues/6697 -sed '/::kM..SeedBytes/d' -i tensorflow/stream_executor/rng.cc - # Due to statically linking boringssl dependency, we have to define one extra # flag when compiling for memory fuzzing (see the boringssl project). if [ "$SANITIZER" = "memory" ]