From 03b010b89022865c0d8c71f5afc70c6faec6f5ad Mon Sep 17 00:00:00 2001 From: Danny Jones <51742311+WittierDinosaur@users.noreply.github.com> Date: Sat, 19 Apr 2025 12:09:45 +0100 Subject: [PATCH] Set up Docker development environment (#6826) --- CONTRIBUTING.md | 32 +++++++++---------- Makefile | 26 +++++++++++++++ .../docker-compose.yml => docker-compose.yml | 25 ++++++++------- .../development/Dockerfile | 17 +++++----- plugins/sqlfluff-templater-dbt/docker/init | 5 --- plugins/sqlfluff-templater-dbt/docker/shell | 3 -- .../sqlfluff-templater-dbt/docker/shutdown | 3 -- plugins/sqlfluff-templater-dbt/docker/startup | 9 ------ requirements_dev.txt | 1 + 9 files changed, 66 insertions(+), 55 deletions(-) create mode 100644 Makefile rename plugins/sqlfluff-templater-dbt/docker/docker-compose.yml => docker-compose.yml (59%) rename plugins/sqlfluff-templater-dbt/docker/Dockerfile.dev => docker/development/Dockerfile (60%) delete mode 100755 plugins/sqlfluff-templater-dbt/docker/init delete mode 100755 plugins/sqlfluff-templater-dbt/docker/shell delete mode 100755 plugins/sqlfluff-templater-dbt/docker/shutdown delete mode 100755 plugins/sqlfluff-templater-dbt/docker/startup diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e74a6a067..51c8373b7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -128,6 +128,18 @@ Additionally if a dbt virtual environment was specified, you will also have A different dbt plugin can be selected by changing the appropriate file under `constraints` for the desired package and version. +#### Developing in Docker + +To build a simple interactive Docker container, run the following commands: +```shell +make build +make shell +``` +This container installs all Python dependencies, and mounts the project directory into +the container. It installs SQLFluff in editable mode. The nuts and bolts are in +place such that git should work seamlessly inside the container. It'll also install +the dbt templater plugin. + ### Wiki We have a [GitHub wiki](https://github.com/sqlfluff/sqlfluff/wiki) with some @@ -268,24 +280,12 @@ tox -e cov-init,dbt019-py39,cov-report-dbt -- plugins/sqlfluff-templater-dbt For more information on adding and running test cases see the [Parser Test README](test/fixtures/dialects/README.md) and the [Rules Test README](test/fixtures/rules/std_rule_cases/README.md). -#### Running dbt templater tests in Docker Compose +#### Running dbt templater tests in Docker -NOTE: If you prefer, you can develop and debug the dbt templater using a -Docker Compose environment. It's a simple two-container configuration: -* `app`: Hosts the SQLFluff development environment. The host's source - directory is mounted into the container, so you can iterate on code - changes without having to constantly rebuild and restart the container. -* `postgres`: Hosts a transient Postgres database instance. - -Steps to use the Docker Compose environment: -* Install Docker on your machine. -* Run `plugins/sqlfluff-templater-dbt/docker/startup` to create the containers. -* Run `plugins/sqlfluff-templater-dbt/docker/shell` to start a bash session - in the `app` container. - -Inside the container, run: +The development Docker container has the dbt templater plugin installed, +so you can run the tests inside the container. Inside the container, run: ``` -py.test -v plugins/sqlfluff-templater-dbt/test/ +pytest -v plugins/sqlfluff-templater-dbt/test/ ``` ### Pre-Commit Config diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..f610c47f8 --- /dev/null +++ b/Makefile @@ -0,0 +1,26 @@ +.PHONY: help build clean fresh shell start stop + +.DEFAULT_GOAL := help + +help: ## Show this available targets + @grep -E '^[/a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' + +build: ## Build the development container + docker-compose build development + +clean: ## Clean up all containers and images + docker system prune -f + docker-compose stop + docker rmi `docker images -a -q` + +fresh: ## Build the development container from scratch + docker-compose build --no-cache development + +shell: ## Start a bash session in the development container + docker-compose exec development bash + +start: ## Start the development container + docker-compose up -d + +stop: ## Stop the development container + docker-compose stop diff --git a/plugins/sqlfluff-templater-dbt/docker/docker-compose.yml b/docker-compose.yml similarity index 59% rename from plugins/sqlfluff-templater-dbt/docker/docker-compose.yml rename to docker-compose.yml index 6833ff78b..e5a1f0580 100644 --- a/plugins/sqlfluff-templater-dbt/docker/docker-compose.yml +++ b/docker-compose.yml @@ -1,19 +1,22 @@ services: - app: - platform: linux/amd64 + development: build: - context: ../../.. - dockerfile: plugins/sqlfluff-templater-dbt/docker/Dockerfile.dev + context: . + dockerfile: ./docker/development/Dockerfile + environment: + - SSH_AUTH_SOCK=/ssh-agent + - POSTGRES_HOST=postgres volumes: - # Host source code directory - - ../../../:/app/ - - ../test/fixtures/dbt/profiles_yml:/root/.dbt + - .:/app + - ./test/fixtures/dbt/profiles_yml:/root/.dbt + - ~/.gitconfig:/etc/gitconfig + - ~/.ssh:/root/.ssh + - /run/host-services/ssh-auth.sock:/ssh-agent + stdin_open: true + tty: true depends_on: - postgres entrypoint: /bin/bash - environment: - - POSTGRES_HOST=postgres - tty: true postgres: image: postgres:14-bullseye environment: @@ -22,4 +25,4 @@ services: # NOTE: "5432:5432" makes the Postgres server accessible to both the host # developer machine *and* the "app" container in Docker. If you don't want # it available on the host machine, change this to simply "5432". - - 5432:5432 + - "5432:5432" diff --git a/plugins/sqlfluff-templater-dbt/docker/Dockerfile.dev b/docker/development/Dockerfile similarity index 60% rename from plugins/sqlfluff-templater-dbt/docker/Dockerfile.dev rename to docker/development/Dockerfile index 1794fda57..c77587088 100644 --- a/plugins/sqlfluff-templater-dbt/docker/Dockerfile.dev +++ b/docker/development/Dockerfile @@ -1,16 +1,17 @@ -FROM python:3.9-slim-bullseye +FROM python:3.12 -RUN apt update \ -&& apt -y install libpq-dev gcc # Set separate working directory for easier debugging. WORKDIR /app +ENV PYTHONPATH=/app;/app/src +# Install Git +RUN apt-get -y update && apt-get -y install git && apt-get clean + +COPY requirements_dev.txt /requirements_dev.txt + +#Install Dependencies RUN --mount=type=cache,target=/root/.cache/pip pip install --upgrade pip setuptools wheel - -# Install requirements separately -# to take advantage of layer caching. -COPY requirements*.txt . -RUN --mount=type=cache,target=/root/.cache/pip pip install --upgrade -r requirements_dev.txt +RUN --mount=type=cache,target=/root/.cache/pip pip install -r /requirements_dev.txt --upgrade --default-timeout=100 # Set up dbt-related dependencies. RUN --mount=type=cache,target=/root/.cache/pip pip install dbt-postgres diff --git a/plugins/sqlfluff-templater-dbt/docker/init b/plugins/sqlfluff-templater-dbt/docker/init deleted file mode 100755 index 078faa5ae..000000000 --- a/plugins/sqlfluff-templater-dbt/docker/init +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -pip install --no-dependencies -e . -e plugins/sqlfluff-templater-dbt -pushd plugins/sqlfluff-templater-dbt/test/fixtures/dbt/dbt_project -dbt deps -popd diff --git a/plugins/sqlfluff-templater-dbt/docker/shell b/plugins/sqlfluff-templater-dbt/docker/shell deleted file mode 100755 index 10bbc8f18..000000000 --- a/plugins/sqlfluff-templater-dbt/docker/shell +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash -my_path="$( cd "$(dirname "$0")"; pwd -P)" -docker compose -f ${my_path}/docker-compose.yml exec app bash diff --git a/plugins/sqlfluff-templater-dbt/docker/shutdown b/plugins/sqlfluff-templater-dbt/docker/shutdown deleted file mode 100755 index 8544bf4a6..000000000 --- a/plugins/sqlfluff-templater-dbt/docker/shutdown +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash -my_path="$( cd "$(dirname "$0")"; pwd -P)" -docker compose -f ${my_path}/docker-compose.yml down -v --remove-orphans diff --git a/plugins/sqlfluff-templater-dbt/docker/startup b/plugins/sqlfluff-templater-dbt/docker/startup deleted file mode 100755 index 866354fe4..000000000 --- a/plugins/sqlfluff-templater-dbt/docker/startup +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash -set -ex -export COMPOSE_DOCKER_CLI_BUILD=1 -export DOCKER_BUILDKIT=1 -my_path="$( cd "$(dirname "$0")"; pwd -P)" -${my_path}/shutdown -docker compose -f ${my_path}/docker-compose.yml build -docker compose -f ${my_path}/docker-compose.yml up -d -docker compose -f ${my_path}/docker-compose.yml exec app "/app/plugins/sqlfluff-templater-dbt/docker/init" diff --git a/requirements_dev.txt b/requirements_dev.txt index 27e42ed60..891496dc7 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -23,6 +23,7 @@ hypothesis pytest pytest-cov pytest-xdist +tox # MyPy # ----