From 1305a75941c7380974d875f5671c828de5fdd8fb Mon Sep 17 00:00:00 2001 From: Carlos O'Ryan Date: Wed, 17 Feb 2021 19:23:43 +0000 Subject: [PATCH 1/2] feat: how-to guide using Cloud Build for builder --- build_scripts/pack/cloudbuild.yaml | 81 +++++++++ .../pack/generate-builder-for-project.sh | 165 ++++++++++++++++++ .../howto_offload_builder_creation/README.md | 148 ++++++++++++++++ 3 files changed, 394 insertions(+) create mode 100644 build_scripts/pack/cloudbuild.yaml create mode 100755 build_scripts/pack/generate-builder-for-project.sh create mode 100644 examples/site/howto_offload_builder_creation/README.md diff --git a/build_scripts/pack/cloudbuild.yaml b/build_scripts/pack/cloudbuild.yaml new file mode 100644 index 00000000..54024e56 --- /dev/null +++ b/build_scripts/pack/cloudbuild.yaml @@ -0,0 +1,81 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +timeout: 3600s +options: + machineType: 'N1_HIGHCPU_32' + diskSizeGb: "512" + +steps: + # Create a docker image for the buildpacks `pack` tool + - name: 'gcr.io/cloud-builders/docker' + args: ['build', '-t', 'pack', '-f', 'build_scripts/pack.Dockerfile', 'build_scripts'] + + # Workaround a kaniko bug using the "edge" builder: + # https://github.com/GoogleContainerTools/kaniko/issues/1058 + # Create the docker images for the buildpacks builder. + - name: 'gcr.io/kaniko-project/executor:edge' + args: [ + "--context=dir:///workspace/", + "--dockerfile=build_scripts/Dockerfile", + "--cache=true", + "--cache-repo=gcr.io/${PROJECT_ID}/functions-framework-cpp/cache", + "--target=gcf-cpp-runtime", + "--destination=gcr.io/${PROJECT_ID}/functions-framework-cpp/run-image", + ] + waitFor: ['-'] + timeout: 1800s + - name: 'gcr.io/cloud-builders/docker' + args: ['pull', 'gcr.io/${PROJECT_ID}/functions-framework-cpp/run-image'] + + - name: 'gcr.io/kaniko-project/executor:edge' + args: [ + "--context=dir:///workspace/", + "--dockerfile=build_scripts/Dockerfile", + "--cache=true", + "--cache-repo=gcr.io/${PROJECT_ID}/functions-framework-cpp/cache", + "--target=gcf-cpp-develop", + "--destination=gcr.io/${PROJECT_ID}/functions-framework-cpp/build-image", + ] + waitFor: ['-'] + timeout: 1800s + - name: 'gcr.io/cloud-builders/docker' + args: ['pull', 'gcr.io/${PROJECT_ID}/functions-framework-cpp/build-image'] + + - name: 'ubuntu' + waitFor: ['-'] + volumes: + - name: 'builder-spec' + path: '/builder' + args: ['build_scripts/pack/generate-builder-for-project.sh', '${PROJECT_ID}', '/builder'] + + - name: 'pack' + volumes: + - name: 'builder-spec' + path: '/builder' + args: ['builder', 'create', 'gcr.io/${PROJECT_ID}/functions-framework-cpp/builder:bionic', '--config', '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/builder/builder.toml'] + + - name: 'pack' + args: ['build', 'test-builder', + '--builder', 'gcr.io/${PROJECT_ID}/functions-framework-cpp/builder:bionic', + '--env', 'FUNCTION_SIGNATURE_TYPE=http', + '--env', 'TARGET_FUNCTION=HelloWorld', + '--path', 'examples/hello_world', + ] + +images: [ + 'gcr.io/${PROJECT_ID}/functions-framework-cpp/run-image', + 'gcr.io/${PROJECT_ID}/functions-framework-cpp/build-image', + 'gcr.io/${PROJECT_ID}/functions-framework-cpp/builder:bionic' +] diff --git a/build_scripts/pack/generate-builder-for-project.sh b/build_scripts/pack/generate-builder-for-project.sh new file mode 100755 index 00000000..ff823983 --- /dev/null +++ b/build_scripts/pack/generate-builder-for-project.sh @@ -0,0 +1,165 @@ +#!/usr/bin/env bash +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu + +if [[ "$#" -ne 2 ]]; then + echo 2> "Usage: $(basename "$0") " + exit 1 +fi + +readonly PROJECT_ID="${1}" +readonly DEST="${2}" + +mkdir -p "${DEST}/buildpack/bin" + +cat >"${DEST}/builder.toml" <<_EOF_ +[[buildpacks]] +uri = "buildpack" + +[[order]] + [[order.group]] + id = "com.google.buildpack.cpp.project" + version = "0.4.0" + +# Stack that will be used by the builder +[stack] +id = "google" +run-image = "gcr.io/${PROJECT_ID}/functions-framework-cpp/run-image:latest" +build-image = "gcr.io/${PROJECT_ID}/functions-framework-cpp/build-image:latest" +_EOF_ + +cat >"${DEST}/buildpack/buildpack.toml" <<_EOF_ +api = "0.2" + +[buildpack] +id = "com.google.buildpack.cpp.project" +version = "0.4.0" +name = "Functions Framework C++ Buildpack for GCP Project ${PROJECT_ID}" + +[[stacks]] +id = "google" +_EOF_ + +cat >"${DEST}/buildpack/bin/detect" <<'_EOF_' +#!/bin/bash + +set -eu + +if ! (compgen -G "*.cc" >/dev/null || + compgen -G "*.cpp" >/dev/null || + compgen -G "*.cxx" >/dev/null || + [[ ! -f CMakeLists.txt ]]); then + exit 100 +fi + +exit 0 +_EOF_ + +cat >"${DEST}/buildpack/bin/build" <<'_SCRIPT_EOF_' +#!/bin/bash + +set -eu + +set -eu + +echo "---> Functions Framework C++ Buildpack" + +layers="$1" + +echo "-----> Setup vcpkg" +export VCPKG_DEFAULT_BINARY_CACHE="${layers}/vcpkg-cache" +export VCPKG_DEFAULT_TRIPLET=x64-linux-nodebug +export VCPKG_ROOT="${layers}/vcpkg" + +if [[ ! -d "${VCPKG_ROOT}" ]]; then + echo "-----> Install vcpkg" + mkdir -p "${VCPKG_ROOT}" + curl -sSL https://github.com/Microsoft/vcpkg/archive/65e5ea1df685a5362e70367bef4dbf827addff31.tar.gz | \ + tar -C "${VCPKG_ROOT}" -xzf - --strip-components=1 + cat >"${VCPKG_ROOT}/triplets/x64-linux-nodebug.cmake" <<_EOF_ +set(VCPKG_BUILD_TYPE release) +set(VCPKG_CMAKE_SYSTEM_NAME Linux) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE static) +set(VCPKG_TARGET_ARCHITECTURE x64) +_EOF_ + cp -r /usr/local/bin/vcpkg "${VCPKG_ROOT}" +cat >"${layers}/vcpkg.toml" < Restore cache from build image" + cp -r /var/cache/vcpkg-cache "${layers}/vcpkg-cache" +cat >"${layers}/vcpkg-cache.toml" < Setup build directory" +cat >"${layers}/source.toml" <"${layers}/source/vcpkg.json" <<_EOF_ +{ + "name": "auto-generated-vcpkg-json", + "version-string": "unversioned", + "dependencies": [ "functions-framework-cpp" ] +} +_EOF_ +fi +/usr/local/share/gcf/build_scripts/generate-wrapper.sh \ + "${TARGET_FUNCTION}" "${FUNCTION_SIGNATURE_TYPE:-http}" >"${layers}/source/main.cc" + +echo "-----> Configure Function" +cat >"${layers}/binary.toml" <"${layers}/local.toml" <"${layers}/launch.toml" < +#include +#include + +namespace gcf = ::google::cloud::functions; + +gcf::HttpResponse hello_world_http(gcf::HttpRequest request) { + auto greeting = [r = std::move(request)] { + auto request_json = nlohmann::json::parse(r.payload(), /*cb=*/nullptr, + /*allow_exceptions=*/false); + if (request_json.count("name") && request_json["name"].is_string()) { + return "Hello " + request_json.value("name", "World") + "!"; + } + return std::string("Hello World!"); + }; + + gcf::HttpResponse response; + response.set_payload(greeting()); + response.set_header("content-type", "text/plain"); + return response; +} +``` + +## Building a Docker image + +Build a Docker image from your function using this buildpack: + +```shell +pack build \ + --builder gcr.io/${GOOGLE_CLOUD_PROJECT}/functions-framework-cpp/builder:bionic \ + --env FUNCTION_SIGNATURE_TYPE=http \ + --env TARGET_FUNCTION=hello_world_http \ + --path examples/site/hello_world_http \ + gcf-cpp-hello-world-http +``` + +To avoid typing the long builder name you can make this your default +builder: + +```shell +pack config default-builder \ + gcr.io/${GOOGLE_CLOUD_PROJECT}/functions-framework-cpp/builder:bionic +``` + +Then the command becomes: + +```shell +pack build \ + --env FUNCTION_SIGNATURE_TYPE=http \ + --env TARGET_FUNCTION=hello_world_http \ + --path examples/site/hello_world_http \ + gcf-cpp-hello-world-http +``` + +## Cleanup + +Delete the images created by Cloud Build: + +```shell +for image in \ + gcr.io/${GOOGLE_CLOUD_PROJECT}/functions-framework-cpp/{builder,cache,build-image,run-image}; do + gcloud container images list-tags "${image}" --format='get(digest)' | \ + xargs printf "${image}@%s\n" | \ + xargs gcloud container images delete --force-delete-tags +done +``` From df242af5aef4e2c3e40a3918f80137876f35cc86 Mon Sep 17 00:00:00 2001 From: Carlos O'Ryan Date: Wed, 17 Feb 2021 22:12:16 +0000 Subject: [PATCH 2/2] Address review comments --- .../pack/generate-builder-for-project.sh | 33 ++++++++++--------- .../site/howto_create_container/README.md | 6 ++-- .../site/howto_deploy_cloud_event/README.md | 9 +++-- .../site/howto_deploy_to_cloud_run/README.md | 9 +++-- .../howto_offload_builder_creation/README.md | 8 ++--- 5 files changed, 32 insertions(+), 33 deletions(-) diff --git a/build_scripts/pack/generate-builder-for-project.sh b/build_scripts/pack/generate-builder-for-project.sh index ff823983..d5501bac 100755 --- a/build_scripts/pack/generate-builder-for-project.sh +++ b/build_scripts/pack/generate-builder-for-project.sh @@ -31,7 +31,7 @@ uri = "buildpack" [[order]] [[order.group]] - id = "com.google.buildpack.cpp.project" + id = "com.google.buildpack.cpp" version = "0.4.0" # Stack that will be used by the builder @@ -45,7 +45,7 @@ cat >"${DEST}/buildpack/buildpack.toml" <<_EOF_ api = "0.2" [buildpack] -id = "com.google.buildpack.cpp.project" +id = "com.google.buildpack.cpp" version = "0.4.0" name = "Functions Framework C++ Buildpack for GCP Project ${PROJECT_ID}" @@ -67,14 +67,13 @@ fi exit 0 _EOF_ +chmod 755 "${DEST}/buildpack/bin/detect" cat >"${DEST}/buildpack/bin/build" <<'_SCRIPT_EOF_' #!/bin/bash set -eu -set -eu - echo "---> Functions Framework C++ Buildpack" layers="$1" @@ -97,29 +96,29 @@ set(VCPKG_LIBRARY_LINKAGE static) set(VCPKG_TARGET_ARCHITECTURE x64) _EOF_ cp -r /usr/local/bin/vcpkg "${VCPKG_ROOT}" -cat >"${layers}/vcpkg.toml" <"${layers}/vcpkg.toml" <<_EOF_ build = true cache = true launch = false -EOL +_EOF_ fi if [[ ! -d "${layers}/vcpkg-cache" ]]; then echo "-----> Restore cache from build image" cp -r /var/cache/vcpkg-cache "${layers}/vcpkg-cache" -cat >"${layers}/vcpkg-cache.toml" <"${layers}/vcpkg-cache.toml" <<_EOF_ build = true cache = true launch = false -EOL +_EOF_ fi echo "-----> Setup build directory" -cat >"${layers}/source.toml" <"${layers}/source.toml" <<_EOF_ build = true cache = false launch = false -EOL +_EOF_ cp -r /usr/local/share/gcf/build_scripts/cmake "${layers}/source" if [[ -r vcpkg.json ]]; then @@ -137,11 +136,11 @@ fi "${TARGET_FUNCTION}" "${FUNCTION_SIGNATURE_TYPE:-http}" >"${layers}/source/main.cc" echo "-----> Configure Function" -cat >"${layers}/binary.toml" <"${layers}/binary.toml" <<_EOF_ build = true cache = true launch = false -EOL +_EOF_ /usr/local/bin/cmake -S "${layers}/source" -B "${layers}/binary" -GNinja -DCMAKE_MAKE_PROGRAM=/usr/local/bin/ninja \ -DCNB_APP_DIR="${PWD}" \ @@ -151,15 +150,17 @@ EOL -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" /usr/local/bin/cmake --build "${layers}/binary" --target install -cat >"${layers}/local.toml" <"${layers}/local.toml" <<_EOF_ launch = true cache = false build = false -EOL +_EOF_ -cat >"${layers}/launch.toml" <"${layers}/launch.toml" <<_EOF_ [[processes]] type = "web" command = "${layers}/local/bin/function" -EOL +_EOF_ _SCRIPT_EOF_ + +chmod 755 "${DEST}/buildpack/bin/build" diff --git a/examples/site/howto_create_container/README.md b/examples/site/howto_create_container/README.md index 38300422..cd0863cb 100644 --- a/examples/site/howto_create_container/README.md +++ b/examples/site/howto_create_container/README.md @@ -21,12 +21,12 @@ docker run hello-world ``` If needed, use the [online instructions][docker-install] to download and install -this tool. This guide assumes that you have configured [sudoless docker], if +this tool. This guide assumes that you have configured [sudoless docker]. If you prefer replace all `docker` commands below with `sudo docker`. Verify the [pack tool][pack-install] is functional on our workstation. These -instructions were tested with v0.17.0, they should work with newer versions. -Some commands may not work with older versions. +instructions were tested with v0.17.0, although they should work with newer +versions. Some commands may not work with older versions. ```shell pack version diff --git a/examples/site/howto_deploy_cloud_event/README.md b/examples/site/howto_deploy_cloud_event/README.md index af50f37e..dcf0d49c 100644 --- a/examples/site/howto_deploy_cloud_event/README.md +++ b/examples/site/howto_deploy_cloud_event/README.md @@ -3,7 +3,7 @@ [repository-gh]: https://github.com/GoogleCloudPlatform/functions-framework-cpp [howto-create-container]: /examples/site/howto_create_container/README.md [cloud-run-quickstarts]: https://cloud.google.com/run/docs/quickstarts -[gcp-quickstarts]: https://cloud.google.com/gcp/getting-started +[gcp-quickstarts]: https://cloud.google.com/resource-manager/docs/creating-managing-projects [buildpacks]: https://buildpacks.io [docker]: https://docker.com/ [docker-install]: https://store.docker.com/search?type=edition&offering=community @@ -28,13 +28,12 @@ docker run hello-world ``` If needed, use the [online instructions][docker-install] to download and install -this tool. This guide assumes that you have configured [sudoless docker], if +this tool. This guide assumes that you have configured [sudoless docker]. If you prefer replace all `docker` commands below with `sudo docker`. - Verify the [pack tool][pack-install] is functional on our workstation. These -instructions were tested with v0.17.0, they should work with newer versions. -Some commands may not work with older versions. +instructions were tested with v0.17.0, although they should work with newer +versions. Some commands may not work with older versions. ```shell pack version diff --git a/examples/site/howto_deploy_to_cloud_run/README.md b/examples/site/howto_deploy_to_cloud_run/README.md index 358697e5..6d101191 100644 --- a/examples/site/howto_deploy_to_cloud_run/README.md +++ b/examples/site/howto_deploy_to_cloud_run/README.md @@ -3,7 +3,7 @@ [repository-gh]: https://github.com/GoogleCloudPlatform/functions-framework-cpp [howto-create-container]: /examples/site/howto_create_container/README.md [cloud-run-quickstarts]: https://cloud.google.com/run/docs/quickstarts -[gcp-quickstarts]: https://cloud.google.com/gcp/getting-started +[gcp-quickstarts]: https://cloud.google.com/resource-manager/docs/creating-managing-projects [buildpacks]: https://buildpacks.io [docker]: https://docker.com/ [docker-install]: https://store.docker.com/search?type=edition&offering=community @@ -27,13 +27,12 @@ docker run hello-world ``` If needed, use the [online instructions][docker-install] to download and install -this tool. This guide assumes that you have configured [sudoless docker], if +this tool. This guide assumes that you have configured [sudoless docker]. If you prefer replace all `docker` commands below with `sudo docker`. - Verify the [pack tool][pack-install] is functional on our workstation. These -instructions were tested with v0.17.0, they should work with newer versions. -Some commands may not work with older versions. +instructions were tested with v0.17.0, although they should work with newer +versions. Some commands may not work with older versions. ```shell pack version diff --git a/examples/site/howto_offload_builder_creation/README.md b/examples/site/howto_offload_builder_creation/README.md index f1d75889..d36f0aa5 100644 --- a/examples/site/howto_offload_builder_creation/README.md +++ b/examples/site/howto_offload_builder_creation/README.md @@ -2,7 +2,7 @@ [repository-gh]: https://github.com/GoogleCloudPlatform/functions-framework-cpp [cloud-build-quickstarts]: https://cloud.google.com/build/docs/quickstarts -[gcp-quickstarts]: https://cloud.google.com/gcp/getting-started +[gcp-quickstarts]: https://cloud.google.com/resource-manager/docs/creating-managing-projects [buildpacks]: https://buildpacks.io [docker]: https://docker.com/ [docker-install]: https://store.docker.com/search?type=edition&offering=community @@ -34,12 +34,12 @@ docker run hello-world ``` If needed, use the [online instructions][docker-install] to download and install -this tool. This guide assumes that you have configured [sudoless docker], if +this tool. This guide assumes that you have configured [sudoless docker]. If you prefer replace all `docker` commands below with `sudo docker`. Verify the [pack tool][pack-install] is functional on our workstation. These -instructions were tested with v0.17.0, they should work with newer versions. -Some commands may not work with older versions. +instructions were tested with v0.17.0, although they should work with newer +versions. Some commands may not work with older versions. ```shell pack version