Set up Docker development environment (#6826)

CONTRIBUTING.md

@@ -128,6 +128,18 @@ Additionally if a dbt virtual environment was specified, you will also have
 A different dbt plugin can be selected by changing the appropriate file under `constraints`
 for the desired package and version.
 
+#### Developing in Docker
+
+To build a simple interactive Docker container, run the following commands:
+```shell
+make build
+make shell
+```
+This container installs all Python dependencies, and mounts the project directory into
+the container. It installs SQLFluff in editable mode. The nuts and bolts are in
+place such that git should work seamlessly inside the container. It'll also install
+the dbt templater plugin.
+
 ### Wiki
 
 We have a [GitHub wiki](https://github.com/sqlfluff/sqlfluff/wiki) with some
@@ -268,24 +280,12 @@ tox -e cov-init,dbt019-py39,cov-report-dbt -- plugins/sqlfluff-templater-dbt
 
 For more information on adding and running test cases see the [Parser Test README](test/fixtures/dialects/README.md) and the [Rules Test README](test/fixtures/rules/std_rule_cases/README.md).
 
-#### Running dbt templater tests in Docker Compose
+#### Running dbt templater tests in Docker
 
-NOTE: If you prefer, you can develop and debug the dbt templater using a
-Docker Compose environment. It's a simple two-container configuration:
-* `app`: Hosts the SQLFluff development environment. The host's source
-  directory is mounted into the container, so you can iterate on code
-  changes without having to constantly rebuild and restart the container.
-* `postgres`: Hosts a transient Postgres database instance.
-
-Steps to use the Docker Compose environment:
-* Install Docker on your machine.
-* Run `plugins/sqlfluff-templater-dbt/docker/startup` to create the containers.
-* Run `plugins/sqlfluff-templater-dbt/docker/shell` to start a bash session
-  in the `app` container.
-
-Inside the container, run:
+The development Docker container has the dbt templater plugin installed,
+so you can run the tests inside the container. Inside the container, run:
 ```
-py.test -v plugins/sqlfluff-templater-dbt/test/
+pytest -v plugins/sqlfluff-templater-dbt/test/
 ```
 
 ### Pre-Commit Config

Makefile

@@ -0,0 +1,26 @@
+.PHONY: help build clean fresh shell start stop
+
+.DEFAULT_GOAL := help
+
+help: ## Show this available targets
+	@grep -E '^[/a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'
+
+build: ## Build the development container
+	docker-compose build development
+
+clean: ## Clean up all containers and images
+	docker system prune -f
+	docker-compose stop
+	docker rmi `docker images -a -q`
+
+fresh: ## Build the development container from scratch
+	docker-compose build --no-cache development
+
+shell: ## Start a bash session in the development container
+	docker-compose exec development bash
+
+start: ## Start the development container
+	docker-compose up -d
+
+stop: ## Stop the development container
+	docker-compose stop

docker-compose.yml

@@ -1,19 +1,22 @@
 services:
-  app:
-    platform: linux/amd64
+  development:
     build:
-      context: ../../..
-      dockerfile: plugins/sqlfluff-templater-dbt/docker/Dockerfile.dev
+      context: .
+      dockerfile: ./docker/development/Dockerfile
+    environment:
+      - SSH_AUTH_SOCK=/ssh-agent
+      - POSTGRES_HOST=postgres
     volumes:
-      # Host source code directory
-      - ../../../:/app/
-      - ../test/fixtures/dbt/profiles_yml:/root/.dbt
+      - .:/app
+      - ./test/fixtures/dbt/profiles_yml:/root/.dbt
+      - ~/.gitconfig:/etc/gitconfig
+      - ~/.ssh:/root/.ssh
+      - /run/host-services/ssh-auth.sock:/ssh-agent
+    stdin_open: true
+    tty: true
     depends_on:
       - postgres
     entrypoint: /bin/bash
-    environment:
-      - POSTGRES_HOST=postgres
-    tty: true
   postgres:
     image: postgres:14-bullseye
     environment:
@@ -22,4 +25,4 @@ services:
       # NOTE: "5432:5432" makes the Postgres server accessible to both the host
       # developer machine *and* the "app" container in Docker. If you don't want
       # it available on the host machine, change this to simply "5432".
-      - 5432:5432
+      - "5432:5432"

docker/development/Dockerfile

@@ -1,16 +1,17 @@
-FROM python:3.9-slim-bullseye
+FROM python:3.12
 
-RUN apt update \
-&& apt -y install libpq-dev gcc
 # Set separate working directory for easier debugging.
 WORKDIR /app
+ENV PYTHONPATH=/app;/app/src
 
+# Install Git
+RUN apt-get -y update && apt-get -y install git && apt-get clean
+
+COPY requirements_dev.txt /requirements_dev.txt
+
+#Install Dependencies
 RUN --mount=type=cache,target=/root/.cache/pip pip install --upgrade pip setuptools wheel
-
-# Install requirements separately
-# to take advantage of layer caching.
-COPY requirements*.txt .
-RUN --mount=type=cache,target=/root/.cache/pip pip install --upgrade -r requirements_dev.txt
+RUN --mount=type=cache,target=/root/.cache/pip pip install -r /requirements_dev.txt --upgrade --default-timeout=100
 
 # Set up dbt-related dependencies.
 RUN --mount=type=cache,target=/root/.cache/pip pip install dbt-postgres

plugins/sqlfluff-templater-dbt/docker/init

@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-pip install --no-dependencies -e . -e plugins/sqlfluff-templater-dbt
-pushd plugins/sqlfluff-templater-dbt/test/fixtures/dbt/dbt_project
-dbt deps
-popd

plugins/sqlfluff-templater-dbt/docker/shell

@@ -1,3 +0,0 @@
-#!/usr/bin/env bash
-my_path="$( cd "$(dirname "$0")"; pwd -P)"
-docker compose -f ${my_path}/docker-compose.yml exec app bash

plugins/sqlfluff-templater-dbt/docker/shutdown

@@ -1,3 +0,0 @@
-#!/usr/bin/env bash
-my_path="$( cd "$(dirname "$0")"; pwd -P)"
-docker compose -f ${my_path}/docker-compose.yml down -v --remove-orphans

plugins/sqlfluff-templater-dbt/docker/startup

@@ -1,9 +0,0 @@
-#!/usr/bin/env bash
-set -ex
-export COMPOSE_DOCKER_CLI_BUILD=1
-export DOCKER_BUILDKIT=1
-my_path="$( cd "$(dirname "$0")"; pwd -P)"
-${my_path}/shutdown
-docker compose -f ${my_path}/docker-compose.yml build
-docker compose -f ${my_path}/docker-compose.yml up -d
-docker compose -f ${my_path}/docker-compose.yml exec app "/app/plugins/sqlfluff-templater-dbt/docker/init"

requirements_dev.txt

@@ -23,6 +23,7 @@ hypothesis
 pytest
 pytest-cov
 pytest-xdist
+tox
 
 # MyPy
 # ----