From 4d4183941fe92423d31e9e9d3b010101ca1b7e5d Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Fri, 2 Apr 2021 16:29:37 -0400 Subject: [PATCH 1/6] chore(devops): move container example under docker dir Signed-off-by: Eric Swanson --- contrib/{make => docker/devops}/.editorconfig | 0 contrib/{make => docker/devops}/Dockerfile | 0 contrib/{make => docker/devops}/README.md | 0 contrib/{make => docker/devops}/makefile | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename contrib/{make => docker/devops}/.editorconfig (100%) rename contrib/{make => docker/devops}/Dockerfile (100%) rename contrib/{make => docker/devops}/README.md (100%) rename contrib/{make => docker/devops}/makefile (100%) diff --git a/contrib/make/.editorconfig b/contrib/docker/devops/.editorconfig similarity index 100% rename from contrib/make/.editorconfig rename to contrib/docker/devops/.editorconfig diff --git a/contrib/make/Dockerfile b/contrib/docker/devops/Dockerfile similarity index 100% rename from contrib/make/Dockerfile rename to contrib/docker/devops/Dockerfile diff --git a/contrib/make/README.md b/contrib/docker/devops/README.md similarity index 100% rename from contrib/make/README.md rename to contrib/docker/devops/README.md diff --git a/contrib/make/makefile b/contrib/docker/devops/makefile similarity index 100% rename from contrib/make/makefile rename to contrib/docker/devops/makefile From 8ea35908238fdb7fd4f05604ab16431fddbc6e3b Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Mon, 5 Apr 2021 10:13:34 -0400 Subject: [PATCH 2/6] chore: vocabulary updates Signed-off-by: Eric Swanson --- .github/styles/vocab.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/styles/vocab.txt b/.github/styles/vocab.txt index 6e3d69330a..e339c256a4 100644 --- a/.github/styles/vocab.txt +++ b/.github/styles/vocab.txt @@ -43,6 +43,7 @@ OAuth Okta Onboarding Platformize +Podman Preprarer Protobuf Proxying @@ -152,6 +153,7 @@ lunr magiclink mailto maintainership +makefile md microsite middleware @@ -257,3 +259,4 @@ www xyz yaml zoomable +zsh From 298ea04df6dbab479d0753bc43b1d33e8c1fd669 Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Mon, 5 Apr 2021 10:14:00 -0400 Subject: [PATCH 3/6] docs(contrib): docker Signed-off-by: Eric Swanson --- contrib/docker/README.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 contrib/docker/README.md diff --git a/contrib/docker/README.md b/contrib/docker/README.md new file mode 100644 index 0000000000..50c291afa0 --- /dev/null +++ b/contrib/docker/README.md @@ -0,0 +1,3 @@ +# Docker contributions + +Here you will find a variety of contributions that rely on Docker as a primary requirement. From bd001c2093b4ff48169997f8457674dadfc23821 Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Mon, 5 Apr 2021 10:15:24 -0400 Subject: [PATCH 4/6] fix(contrib/docker/devops): use libsecret-dev chore: parameterize the vale url Signed-off-by: Eric Swanson --- contrib/docker/devops/Dockerfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/contrib/docker/devops/Dockerfile b/contrib/docker/devops/Dockerfile index e6c33d8a8a..9bb80a0a2c 100644 --- a/contrib/docker/devops/Dockerfile +++ b/contrib/docker/devops/Dockerfile @@ -3,6 +3,7 @@ ARG IMAGE_TAG=14-alpine FROM node:${IMAGE_TAG} ARG APK_VIRTUAL_NAME=.backstage +ARG VALE_INSTALL_URL="https://github.com/errata-ai/vale/releases/download/v2.10.2/vale_2.10.2_Linux_64-bit.tar.gz" WORKDIR /tmp @@ -40,11 +41,13 @@ RUN apk update \ jpeg-dev \ giflib-dev \ librsvg-dev \ - libsecret \ + libsecret-dev \ py3-docutils \ asciidoctor -RUN wget -qO- https://github.com/errata-ai/vale/releases/download/v2.10.2/vale_2.10.2_Linux_64-bit.tar.gz \ +# assumes that "./vale" is in the downloaded archive +# extracts "./vale" and moves it to "/bin" as executable +RUN wget -qO- ${VALE_INSTALL_URL} \ | tar xvz -C /bin vale \ && chmod +x /bin/vale From a133c273e372791ffc9e00694830717f69fd8c9f Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Mon, 5 Apr 2021 10:16:49 -0400 Subject: [PATCH 5/6] chore(contrib/docker/devops): improve make targets more holistic and consistent tasks improved documentation Signed-off-by: Eric Swanson --- contrib/docker/devops/makefile | 142 ++++++++++++++++++++++++++++----- 1 file changed, 120 insertions(+), 22 deletions(-) diff --git a/contrib/docker/devops/makefile b/contrib/docker/devops/makefile index 8b69a41507..3ae89a161b 100644 --- a/contrib/docker/devops/makefile +++ b/contrib/docker/devops/makefile @@ -1,15 +1,39 @@ -docker_tag := backstage-make -docker_name_prefix := backstage-make -frontend_port := 3000 -frontend_host_port := 3000 -backend_port := 7000 -backend_host_port := 7000 +# tag for the local docker image +docker_tag := backstage-make -now_date_time_tag = `date +'%Y%m%d%H%M%S'` -my_dir_path = $(dir $(CURDIR)/$(firstword $(words $(MAKEFILE_LIST)),$(MAKEFILE_LIST))) -project_root_dir_path = $(my_dir_path)/../../ +# prefix for running docker containers +# this helps to quickly identify containers by name +docker_name_prefix := backstage-make + +# post mappings from internal docker containers +# to the host computer +frontend_port := 3000 +frontend_host_port := 3000 +backend_port := 7000 +backend_host_port := 7000 + +# path to this "makefile" +my_dir_path = $(dir $(CURDIR)/$(firstword $(words $(MAKEFILE_LIST)),$(MAKEFILE_LIST))) + +# the relative root of the project +project_root_dir_path = $(my_dir_path)/../../../ + +# dynamic tag to uniquely identify when the script was run +# e.g. running docker images are tagged with this +now_date_time_tag = `date +'%Y%m%d%H%M%S'` + +# prefix the name of docker containers +# for easy identification docker_name_timestamp_prefix = $(docker_name_prefix)-$(now_date_time_tag) +# `make` will execute this +.DEFAULT_GOAL:=all + +# the projects end-to-end tasks execute the build prior to the tests +.PHONY: all +all: build check + +# build the docker container image .PHONY: container container: @docker build \ @@ -17,10 +41,23 @@ container: -f ./Dockerfile \ . +# remove the docker container image from local .PHONY: rm-container rm-container: @docker rmi -f $(docker_tag) +# creates an interactive container session +.PHONY: interactive +interactive: container + @docker run --rm -it \ + --name $(docker_name_timestamp_prefix)-$@ \ + --network host \ + -v $(project_root_dir_path):/app \ + -w /app \ + --entrypoint "/bin/ash" \ + $(docker_tag) + +# clean the project .PHONY: clean clean: container @docker run --rm -it \ @@ -32,6 +69,7 @@ clean: container $(docker_tag) \ rm -rf node_modules +# install dependencies .PHONY: install install: container @docker run --rm -it \ @@ -43,8 +81,8 @@ install: container $(docker_tag) \ yarn install -.PHONY: tsc -tsc: install +.PHONY: build-typescript +build-typescript: install @docker run --rm -it \ --name $(docker_name_timestamp_prefix)-$@ \ --network host \ @@ -54,8 +92,9 @@ tsc: install $(docker_tag) \ yarn tsc +# build the project .PHONY: build -build: tsc +build: build-typescript @docker run --rm -it \ --name $(docker_name_timestamp_prefix)-$@ \ --network host \ @@ -65,8 +104,22 @@ build: tsc $(docker_tag) \ yarn build -# "check" is a standard make target -# using make's terminology instead of the project's +# auto-format a document properly +.PHONY: format-doc +format-doc: install +ifndef path + $(error Missing required "path" argument) +endif + @docker run --rm -it \ + --name $(docker_name_timestamp_prefix)-$@ \ + --network host \ + -v $(project_root_dir_path):/app \ + -w /app \ + --entrypoint "" \ + $(docker_tag) \ + yarn prettier --write $(path) + +# verify document formatting .PHONY: check-docs check-docs: install @docker run --rm -it \ @@ -78,12 +131,33 @@ check-docs: install $(docker_tag) \ yarn run lint:docs -# check-docs alias -.PHONY: lint-docs -lint-docs: check-docs +# analyze the code +.PHONY: check-code +check-code: install + @docker run --rm -it \ + --name $(docker_name_timestamp_prefix)-$@ \ + --network host \ + -v $(project_root_dir_path):/app \ + -w /app \ + --entrypoint "" \ + $(docker_tag) \ + yarn run lint:all -.PHONY: check-prettier -check-prettier: install +.PHONY: check-type-dependencies +check-type-dependencies: install + @docker run --rm -it \ + --name $(docker_name_timestamp_prefix)-$@ \ + --network host \ + -v $(project_root_dir_path):/app \ + -w /app \ + --entrypoint "" \ + $(docker_tag) \ + yarn run lint:type-deps + +# check is a standard target for make +# so, prefixing with "check-" +.PHONY: check-styles +check-styles: install @docker run --rm -it \ --name $(docker_name_timestamp_prefix)-$@ \ --network host \ @@ -93,12 +167,36 @@ check-prettier: install $(docker_tag) \ yarn run prettier:check +# execute all tests +# BUG: failing tests +# https://github.com/backstage/backstage/issues/5212 +.PHONY: check-tests +check-tests: build + @docker run --rm -it \ + --name $(docker_name_timestamp_prefix)-$@ \ + --network host \ + -v $(project_root_dir_path):/app \ + -w /app \ + --entrypoint "" \ + $(docker_tag) \ + yarn run test:all + +# convenience: dev alias +.PHONY: test +test: check-tests + +# check all the things +# BUG: check-tests failing (see target comments) +.PHONY: check +check: check-code check-docs check-type-dependencies check-styles + +# run development instance # BUG: the frontend seems to run on "$(backend_port)" (7000 default). # The documentation states "This is going to start two things, # the frontend (:3000) and the backend (:7000)." # However, the frontend seems to end up running on 7000. .PHONY: dev -dev: +dev: build @docker run --rm -it \ --name $(docker_name_timestamp_prefix)-$@ \ -p $(frontend_port):$(frontend_host_port) \ @@ -110,10 +208,10 @@ dev: $(docker_tag) \ yarn dev -# dev alias +# convenience: dev alias .PHONY: start start: dev -# dev alias +# convenience: dev alias .PHONY: run run: dev From 3067de45a86e82121c3ca55e57251bb266900659 Mon Sep 17 00:00:00 2001 From: Eric Swanson Date: Mon, 5 Apr 2021 10:18:12 -0400 Subject: [PATCH 6/6] docs(contrib/docker/devops): improve devops containers readability Signed-off-by: Eric Swanson --- contrib/docker/devops/README.md | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/contrib/docker/devops/README.md b/contrib/docker/devops/README.md index f1cc87b86d..c492ce92de 100644 --- a/contrib/docker/devops/README.md +++ b/contrib/docker/devops/README.md @@ -1,12 +1,18 @@ -# Immutable Task Pattern +# DevOps Containers -Whether executed locally on an engineer's machine or remotely on a server, ensuring consistent DevOps task execution improves reproducibility and the quality of an experiment. +End-to-end DevOps extends from local development to post-production operations. Whether executed locally on an engineer's machine or remotely on a server, ensuring consistent DevOps task execution improves reproducibility and the quality of an experiment. + +Goals of this "contrib" project include: + +- Reduce Backstage contributor prerequisites to a single container runtime (e.g. Podman, Docker, etc.) +- Provide examples of shared, immutable task execution environments for local development that can also be used by remote DevOps Continuous Integration and Deployment (CI/CD) +- Give ideas to the core project for opportunities to improve ## Context and problem Great progress has been made in developing immutable execution environments for application deployments as well as for the build, test, and deploy environments themselves (continuous integration, deployment and release). Local development can also benefit from immutable execution environments, especially when replicating issues. -Engineers typically execute tasks directly from their machine. As a result, they have different operating systems, program and package versions, and configurations. While locking versions of application package dependencies and even tooling with modern features, like `yarn set version ...`, help, there can still be significant differences from machine to machine. +Engineers typically execute tasks directly from their machine. As a result, they have different operating systems, program and package versions, and configurations. While locking versions of application package dependencies and even tooling with modern version "pinning" features help, like `yarn set version ...`, there can still be significant differences from machine to machine. ## Solution @@ -15,24 +21,27 @@ Containers, like [Docker](https://www.docker.com/), enable an immutable definiti ## Issues and considerations - Hosted and sometimes proprietary tasks may be unavoidable, like those offered in the GitHub Actions marketplace or tasks built into products like the core capabilities of Jenkins or through plug-ins. These tasks typically require a "poke the server" approach to debugging end-to-end workflows or containerizing the installation and configuration of the products themselves to be executed as larger containerized tasks. This might introduce the need for guidance on when to define containerized tasks and when to lean on hosted and/or proprietary solutions. -- Cross-platform presents challenges. Getting consistency across Linux, macOS, and Windows requires knowledge and effort. A quick argument can be to standardize on Linux shell across these three major platforms, but even working across Linux distributions presents some unique challenges too. +- Cross-platform presents challenges. Getting consistency across operating systems like Linux, macOS, and Windows requires knowledge and effort. A suggestion might be to standardize on a common shell across these three major platforms, since macOS supports bash and is moving the zsh, Linux supports both of those and Windows users can run the Subsystem for Linux and/or Hyper-V and/or Docker images. Choosing a consistent operating system and shell for the container can be problematic. - Consider the balance of developers freedom of choice in tooling with the consistency with standardized tooling that can be achieved with containerized execution environments. Empower developers to "opt-out" and swap tooling choices wherever possible to experiment and innovate. +- Developers are not likely to agree on solutions unique to a single development editor. Ideally, a solution should be capable of supporting a variety of development tools without requiring them. +- This space is rapidly evolving. Cloud development environments like [GitPod](https://www.gitpod.io/), [VS Code Remote Development](https://code.visualstudio.com/docs/remote/remote-overview), or [Eclipse Che](https://www.eclipse.org/che/) and [Che-Theia](https://github.com/eclipse-che/che-theia). +- Debugging can be tricky, especially when running and stepping through code for breakpoints. Debugging solutions may be specific to a product. For example, VS Code debugging with the remote development extension. ## When to use this pattern - To ensure task execution consistency across a variety of environments, especially across developer workstations and remote DevOps servers - For tasks that are particularly fragile and/or complex (e.g. developers consistently reporting setup and environmental issues) -- Code editors and "Integrated Development Environments" may offer support for containerized development, like Visual Studio Code's Remote Development extension. This pattern can easily be extended to support these scenarios. One important consideration is GUI tooling support. Most task execution shouldn't require a GUI, especially for remote DevOps tasks. However, developers might use GUI tooling that could also use containerization or virtualization. So, consider how to achieve both. For example "Docker in Docker" (DiD) could enable a GUI development container to execute containerized tasks in separate containers. +- Code editors may offer support for containerized development. This pattern can easily be extended to support these scenarios. One important consideration is GUI tooling support. Most task execution shouldn't require a GUI, especially for remote DevOps tasks. However, developers might use GUI tooling that could also use containerization or virtualization. So, consider how to achieve both. For example "Docker in Docker" (DiD) could enable a GUI development container to execute containerized tasks in separate containers. ## Example -Backstage is a web application with prerequisites on Node and Yarn. Engineers are encouraged to install a specific version of Node and Yarn. The version of node is partially enforced by the "engines" configuration in "package.json". And, the version of `yarn` was "locked" using the command `yarn set version ...`, resulting in the root project yarn configuration file with the setting "yarn-path" that contains a specific version of Yarn. Engineers are encouraged to deploy with Docker. +Backstage is a web application with prerequisites on Node and Yarn. Engineers are encouraged to install a specific version of Node and Yarn. The version of node is partially enforced by the "engines" configuration in "package.json". And, the version of `yarn` was "pinned" using the command `yarn set version ...`, resulting in the root project yarn configuration file with the setting "yarn-path" that contains a specific version of Yarn. Engineers are encouraged to deploy with Docker for testing. While local execution of DevOps tasks are defined in the "scripts" section of Node's "package.json" file, the DevOps tasks used during continuous integration and deployment are defined separately in GitHub workflow YAML files to be executed by GitHub Actions using separate task definitions. Docker could be used to unite the execution of local workstation tasks and remote DevOps tasks. With a consistent container image definition, tasks can be executed with guarantees about the OS, tooling and versions and configuration. -A simple option might be to build the image as needed. Local development could build the image once and reuse it over time. However, DevOps servers would need to build the image on every execution set. Also, building ad-hoc still introduces some significant variability when tooling is installed into the container and the system is updated. +A simple option might be to build the image as needed. Local development could build the image once and reuse it over time. However, DevOps servers would need to build the image on every execution set. Each build introduces some significant variability when tooling is installed into the container and the system is updated, so it is important to lock tooling versions as they are installed. A more robust option might be to define and build the DevOps container(s) for task execution once and then reuse the container as needed. Local development could pull the exact same image that remote DevOps servers also pull. @@ -40,7 +49,7 @@ This example will implement the simple option and leave it to project adopters t ### Prerequisites -Because the environments are containerized, the only requirement is [Docker](https://docs.docker.com/get-docker/). Please install the latest stable version if possible. Docker is simply one of the more ubiquitous runtime options. However, you might also prefer any runtime that supports building and running docker images. If a pre-built image were build and made available using a standard like the [Open Container Initiative](https://opencontainers.org/), then you could use any OCI runtime. +Because the environments are containerized, the only requirement is [Docker](https://docs.docker.com/get-docker/) or [Podman](https://podman.io/). Please install the latest stable version if possible. Docker is simply one of the more ubiquitous runtime options. However, you might also prefer any runtime that supports building and running docker images. If a pre-built image were build and made available using a standard like the [Open Container Initiative](https://opencontainers.org/), then you could use any OCI runtime. [GNU Make](https://www.gnu.org/software/make/) is highly recommended to ensure the docker commands execute consistently. Microsoft Windows users can install a Linux shell with `make` installed (e.g. Windows Subsystem for Linux, Cygwin, etc.). It would also be possible to define shell scripts and remove the `make` dependency. However, GNU Make was built for this very purpose. Without it, each `make` command will need to be understood, built (i.e. manual variable substitution) and executed. @@ -49,8 +58,8 @@ Because the environments are containerized, the only requirement is [Docker](htt The steps below attempt to mirror the exact same onboarding steps for a new contributor. 1. Open a shell and change to this directory: `cd contrib/make` -2. `make install` will build the container and then execute the project's "install" setup including `npm` dependencies -3. `make tsc` will compile typescript -4. `make build` will build the project +2. `make` to will build the container, execute the project's "install" setup including `npm` dependencies, check documents, code and styles, execute the tests, compile typescript and build the project + +See the "makefile" for more command targets to execute from your host machine. For an interactive shell session within the container, run `make interactive`. If this were a standard for implementation, all tasks, including local development and remote DevOps, would be defined in the same way. For example, GitHub Actions would rely on these commands and containers to execute the same tasks a developer could execute locally with the exception of using 3rd party tasks from the GitHub Actions marketplace.