fixes #6463: Set Variable Parsing for SparkSQL and Databricks (#6464)

Co-authored-by: F STG <fritz.steingrube@extern.stromnetz-hamburg.de>

.deepsource.toml

@@ -1,14 +0,0 @@
-version = 1
-
-test_patterns = [
-  'test/**',
-]
-
-exclude_patterns = [
-  'docs/**',
-  'util.py', # not part of the core sqlfluff code
-]
-
-[[ analyzers ]]
-  name = 'python'
-  enabled = true

.editorconfig

@@ -14,10 +14,40 @@ indent_size = 2
 [*.py]
 indent_size = 4
 
-# Don't correct spacing for sql and yaml files as sometimes want them wrong for tests
-[*.sql]
+# Don't correct indentation for sql and yaml files as sometimes want them wrong for tests
+[*.{yml,yaml,sql}]
+indent_style = unset
+
+# Some specific tests with trailing newlines
+# If adding any exceptions here, make sure to add them to .pre-commit-config.yaml as well
+[test/fixtures/templater/jinja_l_metas/0{01,03,04,05,07,08,11}.sql]
+indent_style = unset
+insert_final_newline = unset
+[test/fixtures/linter/sqlfluffignore/*/*.sql]
 indent_style = unset
 trim_trailing_whitespace = unset
+insert_final_newline = unset
+[test/fixtures/config/inheritance_b/{,nested/}example.sql]
+indent_style = unset
+trim_trailing_whitespace = unset
+insert_final_newline = unset
+[trailing_newlines.sql]
+indent_style = unset
+trim_trailing_whitespace = unset
+insert_final_newline = unset
+[plugins/sqlfluff-templater-dbt/test/fixtures/dbt/dbt_project/models/my_new_project/multiple_trailing_newline.sql]
+indent_style = unset
+insert_final_newline = unset
+[plugins/sqlfluff-templater-dbt/test/fixtures/dbt/templated_output/macro_in_macro.sql]
+indent_style = unset
+trim_trailing_whitespace = unset
+insert_final_newline = unset
+[plugins/sqlfluff-templater-dbt/test/fixtures/dbt/templated_output/{,dbt_utils_0.8.0/}last_day.sql]
+indent_style = unset
+trim_trailing_whitespace = unset
+insert_final_newline = unset
+[test/fixtures/linter/indentation_errors.sql,test/fixtures/templater/jinja_d_roundtrip/test.sql]
+trim_trailing_whitespace = false
 
 [*.rst]
 indent_size = 3

.github/FUNDING.yml

@@ -12,4 +12,7 @@
 # as a donation toward infrastructure and hosting costs, or to enable
 # maintainers to spend more time on the project.
 
-custom: flattr.com/github/alanmcruickshank
+# For more details on how this money is used, see the GitHub sponsor
+# page for sqlfluff at https://github.com/sponsors/sqlfluff.
+
+github: sqlfluff

.github/ISSUE_TEMPLATE/enhancement.yml

@@ -4,8 +4,8 @@ labels: [ "enhancement" ]
 body:
   - type: markdown
     attributes:
-      value: |
-        Use this enhancement template to suggest new features or functionality for SQLFluff. Please not
+      value: >
+        Use this enhancement template to suggest new features or functionality for SQLFluff. Please note that
         missing syntax support for any of our currently supported dialects, should instead be filed as a
         [Bug Report](https://github.com/sqlfluff/sqlfluff/issues/new?assignees=&labels=bug&template=bug-report.yml).
 

.github/codecov.yml

@@ -1,25 +0,0 @@
-# Configuration file for codecov.io
-# https://docs.codecov.io/docs/codecovyml-reference
-
-codecov:
-  branch: main
-  bot: "codecov-io"
-  ci:
-    - "github.com"
-  max_report_age: 24
-  disable_default_path_fixes: no  # yamllint disable-line rule:truthy
-  require_ci_to_pass: yes  # yamllint disable-line rule:truthy
-  notify:
-    after_n_builds: 9
-    wait_for_ci: yes  # yamllint disable-line rule:truthy
-
-coverage:
-  precision: 2
-  round: down
-  range: "70...100"
-  status:
-    project:
-      default:
-        # basic
-        target: auto
-        threshold: 0%

.github/labeler.yml

@@ -0,0 +1,77 @@
+ansi:
+  - "/(ansi)/i"
+
+athena:
+  - "/(athena)/i"
+
+bigquery:
+  - "/(bigquery)/i"
+
+clickhouse:
+  - "/(clickhouse)/i"
+
+databricks:
+  - "/(databricks)/i"
+
+db2:
+  - "/(db2)/i"
+
+duckdb:
+  - "/(duckdb)/i"
+
+exasol:
+  - "/(exasol)/i"
+
+greenplum:
+  - "/(greenplum)/i"
+
+hive:
+  - "/(hive)/i"
+
+impala:
+  - "/(impala)/i"
+
+mariadb:
+  - "/(mariadb)/i"
+
+materialize:
+  - "/(materialize)/i"
+
+mysql:
+  - "/(mysql)/i"
+
+oracle:
+  - "/(oracle)/i"
+
+postgres:
+  - "/(postgres)/i"
+
+redshift:
+  - "/(redshift)/i"
+
+snowflake:
+  - "/(snowflake)/i"
+
+soql:
+  - "/(soql)/i"
+
+sparksql:
+  - "/(sparksql)/i"
+
+sqlite:
+  - "/(sqlite)/i"
+
+starrocks:
+  - "/(starrocks)/i"
+
+t-sql:
+  - "/(t-sql|tsql)/i"
+
+teradata:
+  - "/(teradata)/i"
+
+trino:
+  - "/(trino)/i"
+
+vertica:
+  - "/(vertica)/i"

.github/workflows/add-issue-labels.yaml

@@ -0,0 +1,18 @@
+name: "Add Issue Labels"
+on:
+  issues:
+    types: [opened]
+
+jobs:
+  triage:
+    runs-on: ubuntu-latest
+    steps:
+      # Update .github/labeler.yml for new dialects
+      - uses: github/issue-labeler@v3.2
+        with:
+          configuration-path: .github/labeler.yml
+          include-title: 1
+          include-body: 0
+          not-before: 2023-07-06T02:54:32Z
+          enable-versioned-regex: 0
+          repo-token: ${{ github.token }}

.github/workflows/add-to-release-notes.yml

@@ -7,11 +7,16 @@ on:
     branches:
       - main
 jobs:
-  release-notes:
+  draft-release:
     runs-on: ubuntu-20.04
     if: github.repository == 'sqlfluff/sqlfluff'
     steps:
     - name: Update release notes
       uses: release-drafter/release-drafter@v5
+      # If it's a release PR, then we should also update
+      # the header and title here too.
+      # with:
+      #   name: ${{    }}
+      #   header: ${{    }}
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/check-yaml-files-generation.yml

@@ -1,27 +0,0 @@
-name: "Check generated YAML files"
-on: pull_request
-jobs:
-  ymlchecks:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python 3.10
-      uses: actions/setup-python@v2
-      with:
-        python-version: '3.10'
-    - name: Install dependencies
-      run: |
-        pip install -Ur requirements.txt -Ur requirements_dev.txt
-        python setup.py develop
-    - name: Generate the YAML files
-      run: |
-        python test/generate_parse_fixture_yml.py
-    - name: Test the generated YAML files
-      run: |
-        if [ -n "$(git status --porcelain)" ]; then
-          git diff
-          echo "Generated YAML files do not match branch."
-          echo "Please run the following command to generate these:"
-          echo "  python test/generate_parse_fixture_yml.py"
-          exit 1
-        fi

.github/workflows/ci-pr-comments.yml

@@ -0,0 +1,107 @@
+# This Workflow runs in a more secure context and comments
+# on pull requests.
+# https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
+name: Comment on the pull request
+
+# Run on completion of the CI job.
+# This workflow has access to write comments on PRs event when
+# that PR is triggered by a forked repo.
+on:
+  workflow_run:
+    workflows:
+      - CI
+    types:
+      - completed
+
+jobs:
+  comment-on-pr:
+    runs-on: ubuntu-latest
+    if: >
+      github.event.workflow_run.event == 'pull_request'
+    steps:
+      - name: 'Download txt artifact'
+        uses: actions/github-script@v6
+        with:
+          script: |
+            const artifacts = await github.rest.actions.listWorkflowRunArtifacts({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               run_id: ${{github.event.workflow_run.id }},
+            });
+            const matchArtifact = artifacts.data.artifacts.filter((artifact) => {
+              return artifact.name == "txt-report"
+            })[0];
+            const download = await github.rest.actions.downloadArtifact({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               artifact_id: matchArtifact.id,
+               archive_format: 'zip',
+            });
+            var fs = require('fs');
+            fs.writeFileSync('${{github.workspace}}/cov-report.zip', Buffer.from(download.data));
+
+      - name: Unzip Downloaded Artifact
+        run: unzip cov-report.zip
+
+      - name: Update PR comment with coverage report.
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            // First list the existing comments
+            const trigger_str = 'Coverage Results';
+            console.log("Getting existing comments...");
+
+            const { promises: fs } = require('fs');
+            const issue_number = await fs.readFile('pr-number.txt', 'utf8');
+            console.log("Issue number: " + issue_number);
+
+            const comments = await github.paginate(
+              github.rest.issues.listComments,
+              {
+                owner: 'sqlfluff',
+                repo: 'sqlfluff',
+                issue_number: Number(issue_number)
+              }
+            );
+
+            let comment_id = null;
+            console.log("Got %d comments", comments.length);
+
+            comments.forEach(comment => {
+              if (comment.body.indexOf(trigger_str) >= 0) {
+                console.log("Found target comment ID: %d", comment.id);
+                comment_id = comment.id;
+              } else {
+                console.log("Comment ID %d not valid with body:\n%s.", comment.id, comment.body);
+              }
+            });
+
+            const previous_outcome = await fs.readFile('outcome.txt', 'utf8');
+            console.log("Previous coverage step outcome: %s", previous_outcome);
+            if (previous_outcome == "success\n") {
+              status_emoji = "✅";
+            } else {
+              status_emoji = "⚠️";
+            }
+
+            const content = await fs.readFile('coverage-report.txt', 'utf8');
+            body = "# " + trigger_str + " " + status_emoji + "\n```\n" + content + "\n```\n";
+
+            if (comment_id > 0) {
+              console.log("Updating comment id: %d", comment_id);
+              await github.rest.issues.updateComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: comment_id,
+                body: body
+              });
+            } else {
+              console.log("No existing comment matched, creating a new one...");
+              await github.rest.issues.createComment({
+                issue_number: Number(issue_number),
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                body: body
+              });
+            }

.github/workflows/ci-test-dbt.yml

@@ -0,0 +1,86 @@
+#############################
+## GitHub Actions CI Tests ##
+#############################
+#
+# This is a reusable workflow to make CI tests more modular.
+# See: https://docs.github.com/en/actions/using-workflows/reusing-workflows
+#
+# Called by ci-tests.yml
+# This one does the dbt tests
+#
+
+name: Modular SQLFluff dbt test workflow
+
+on:
+  workflow_call:
+    inputs:
+      python-version:
+        required: true
+        type: string
+      dbt-version:
+        required: true
+        type: string
+      coverage:
+        required: false
+        type: boolean
+        default: false
+    secrets:
+      gh_token:
+        required: true
+
+jobs:
+  modular-python-test:
+    name: py${{ inputs.python-version }}-${{ inputs.dbt-version }}
+    runs-on: ubuntu-latest
+    env:
+      FORCE_COLOR: 1
+
+    services:
+      # Label used to access the service container
+      postgres:
+        # Docker Hub image
+        image: postgres
+        # Provide the password for postgres
+        env:
+          POSTGRES_PASSWORD: password
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          # Maps tcp port 5432 on service container to the host
+          - 5432:5432
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ inputs.python-version }}
+        cache: 'pip'
+        cache-dependency-path: |
+          setup.cfg
+          requirements_dev.txt
+
+    - name: Install dependencies
+      run: pip install tox
+
+    - name: Run the tests (with coverage)
+      if: ${{ inputs.coverage }}
+      run: tox -e ${{ inputs.dbt-version }} -- --cov=sqlfluff_templater_dbt plugins/sqlfluff-templater-dbt
+
+    - name: Run the tests (without coverage)
+      if: ${{ !inputs.coverage }}
+      run: tox -e ${{ inputs.dbt-version }} -- plugins/sqlfluff-templater-dbt
+
+    - name: Upload coverage data (github)
+      uses: actions/upload-artifact@v4
+      if: ${{ inputs.coverage }}
+      with:
+        name: coverage-data-py${{ inputs.python-version }}-${{ inputs.dbt-version }}
+        path: ".coverage.*"
+        if-no-files-found: ignore
+        include-hidden-files: true

.github/workflows/ci-test-python.yml

@@ -0,0 +1,90 @@
+#############################
+## GitHub Actions CI Tests ##
+#############################
+#
+# This is a reusable workflow to make CI tests more modular.
+# See: https://docs.github.com/en/actions/using-workflows/reusing-workflows
+#
+# Called by ci-tests.yml
+# This one does the python tests
+#
+
+name: Modular SQLFluff python test workflow
+
+on:
+  workflow_call:
+    inputs:
+      python-version:
+        required: true
+        type: string
+      marks:
+        required: false
+        type: string
+        default: "not integration"
+      coverage:
+        required: false
+        type: boolean
+        default: false
+    secrets:
+      gh_token:
+        required: true
+
+jobs:
+  modular-python-test:
+    runs-on: ubuntu-latest
+    name: py${{ inputs.python-version }}
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ inputs.python-version }}
+        cache: 'pip'
+        cache-dependency-path: |
+          setup.cfg
+          requirements_dev.txt
+
+    - name: Install dependencies
+      run: pip install tox
+
+    - name: Parse Python Version
+      id: py_version
+      run: |
+        PYVERSION=$(echo "${{ inputs.python-version }}" | sed -e 's/\.//g')
+        echo "PYVERSION=$PYVERSION" >> $GITHUB_OUTPUT
+
+    # Run test process (with or without coverage).
+    # Arguments after the "--" are passed through to pytest:
+    #   --cov=...       The library to include in coverage reporting.
+    #   -n 2            Runs with two parallel processes.
+    #   test            The path to detect tests within.
+    #   -m ...          The pytest marks to filter tests.
+    #   --durations=16  Displays the 16 slowest runs to help with performance debugging.
+    - name: Run the tests (with coverage)
+      # NOTE: We have a separate job for coverage reporting because
+      # it impacts performance and slows the test suite significantly.
+      if: ${{ inputs.coverage }}
+      run: tox -e py${{ steps.py_version.outputs.PYVERSION }} -- --cov=sqlfluff -n 2 test -m "${{ inputs.marks }}" --durations=16
+    - name: Run the tests (without coverage)
+      if: ${{ !inputs.coverage }}
+      run: tox -e py${{ steps.py_version.outputs.PYVERSION }} -- -n 2 test -m "${{ inputs.marks }}" --durations=16
+
+    - name: Rename coverage files with suffix
+      # NOTE: We do this because we're using the same tox environment for multiple
+      # test jobs and we need to make sure that their coverage files don't collide.s
+      id: cov_suffix
+      if: ${{ inputs.coverage }}
+      run: |
+        COVSUFFIX=$(echo "${{ inputs.marks }}" | sed -e 's/ /-/g')
+        echo "COVSUFFIX=$COVSUFFIX" >> $GITHUB_OUTPUT
+        for file in .coverage.*; do mv "$file" "$file.$COVSUFFIX"; done;
+
+    - name: Upload coverage data (github)
+      uses: actions/upload-artifact@v4
+      if: ${{ inputs.coverage }}
+      with:
+        name: coverage-data-py${{ inputs.python-version }}-${{ inputs.marks }}
+        path: ".coverage.*"
+        if-no-files-found: ignore
+        include-hidden-files: true

.github/workflows/ci-tests.yml

@@ -2,118 +2,195 @@
 ## GitHub Actions CI Tests ##
 #############################
 #
-# Based on our old CircleCI config
-#
 # This can be kicked off manually in the Actions tab of GitHub
 # It will also run nightly at 2pm
 # It will run on any pull request, except non-code changes
 # (images, markdown files, )
 #
-name: CI Tests
+name: CI
+
 on:
   workflow_dispatch:
   schedule:
     # 2am each night
     - cron: '00 2 * * *'
+  # Don't use pull_request_target here. See:
+  # https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
   pull_request:
   push:
     branches:
       - main
+  merge_group:
+    # Merge Queue checks requested. This feature is still in beta
+    # from Github and so may need updating later.
+    # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#merge_group
+    types: [checks_requested]
 
 jobs:
+
   linting:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        jobs: [ 'linting', 'doclinting', 'docbuild', 'yamllint' ]
-    name: ${{ matrix.jobs }} tests
+        job:
+          [
+            "linting",
+            "doclinting",
+            "docbuild",
+            "yamllint",
+            "mypy",
+            "mypyc",
+            "doctests",
+          ]
+        include:
+          # Default to most recent python version
+          - python-version: "3.13"
+          # As at 2024-10-10, docbuild fails on 3.13, so fall back to 3.12
+          - job: docbuild
+            python-version: "3.12"
+    name: ${{ matrix.job }} tests
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python
-      uses: actions/setup-python@v2
-      with:
-        python-version: '3.10'
-    - name: Install dependencies
-      run: |
-        pip install tox
-    - name: Run the tests
-      run: |
-        tox -e ${{ matrix.jobs }}
-  ruleslinting:
-    runs-on: ubuntu-latest
-    name: Rules critical errors tests
-    steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python
-      uses: actions/setup-python@v2
-      with:
-        python-version: '3.10'
-    - name: Install dependencies
-      run: |
-        pip install tox
-        pip install .
-    - name: Run the tests
-      run: |
-        tox -e ruleslinting
-  python-version-tests:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: [ '3.7', '3.8', '3.9', '3.10' ]
-    name: Python ${{ matrix.python-version }} tests
-    steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
-      run: |
-        pip install tox
-    - name: Set Tox Python Param
-      run: |
-        echo 'TOXENV<<EOF' >> $GITHUB_ENV
-        echo "${{ matrix.python-version }}" | sed "s/\.//" | sed "s/^/py/" >> $GITHUB_ENV
-        echo 'EOF' >> $GITHUB_ENV
+      run: pip install tox
     - name: Run the tests
-      run: |
-        tox -e ${{ env.TOXENV }} -- -n 2 test
-    - name: Upload Coverage Report
-      uses: codecov/codecov-action@v1
-      with:
-        files: ./coverage.xml
-  other-tests:
-    runs-on: ubuntu-latest
+      run: tox -e ${{ matrix.job }}
+
+  # Test with coverage tracking on most recent python (py313).
+  python-version-tests:
+    name: Python Tests
     strategy:
       matrix:
-        jobs: [ 'bench', 'mypy' ]
-    name: ${{ matrix.jobs }} tests
+        python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12", "3.13" ]
+        include:
+          # Default to test without coverage tracking on older python versions.
+          # This saves time, as testing without coverage tracking is faster.
+          - coverage: false
+          # Override coverage to be true for most recent python version.
+          - python-version: "3.13"
+            coverage: true
+    permissions:
+      contents: read
+      pull-requests: write
+    uses: ./.github/workflows/ci-test-python.yml
+    with:
+      python-version: ${{ matrix.python-version }}
+      coverage: ${{ matrix.coverage }}
+    secrets:
+      gh_token: ${{ secrets.github_token }}
+
+  dbt-tests:
+    name: dbt Plugin Tests
+    strategy:
+      fail-fast: false
+      matrix:
+        dbt-version:
+          - dbt140
+          - dbt150
+          - dbt160
+          - dbt170
+          - dbt180
+          - dbt190
+        include:
+          # Default to python 3.12 for dbt tests.
+          # * Python 3.13 not supported yet.
+          # * Looks like it's due to psycopg2 support as of 2024-10-10
+          - python-version: "3.12"
+          # For dbt 1.4 - 1.6 override to python 3.11
+          - dbt-version: dbt140
+            python-version: "3.11"
+          - dbt-version: dbt150
+            python-version: "3.11"
+          - dbt-version: dbt160
+            python-version: "3.11"
+    permissions:
+      contents: read
+      pull-requests: write
+    uses: ./.github/workflows/ci-test-dbt.yml
+    with:
+      python-version: ${{ matrix.python-version }}
+      dbt-version: ${{ matrix.dbt-version }}
+      coverage: true
+    secrets:
+      gh_token: ${{ secrets.github_token }}
+
+  dialect-tests:
+    name: Dialect ${{ matrix.marks }}
+    strategy:
+      matrix:
+        include:
+          # This runs the bulk of the dialect _parsing_ tests.
+          #
+          # It's run as a separate job as takes longer than the CI jobs and allows
+          # them to be rerun separately if GitHub Actions or Coverage is experiencing
+          # issues.
+          - marks: "parse_suite"
+            # We test coverage here for some parsing routines.
+            coverage: true
+          # This lints all our dialect fixtures to check rules can handle a variety
+          # of SQL and don't error out badly.
+          #
+          # It's run as a separate job as takes longer than the CI jobs and allows
+          # them to be rerun separately if GitHub Actions or Coverage is experiencing
+          # issues.
+          - marks: "fix_suite"
+            coverage: false
+          # This lints all our rules fixtures to check rules.
+          #
+          # It's run as a separate job as takes longer than the CI jobs and allows
+          # them to be rerun separately if GitHub Actions or Coverage is experiencing
+          # issues.
+          - marks: "rules_suite"
+            coverage: true
+    permissions:
+      contents: read
+      pull-requests: write
+    uses: ./.github/workflows/ci-test-python.yml
+    with:
+      python-version: "3.13"
+      marks: ${{ matrix.marks }}
+      coverage: ${{ matrix.coverage }}
+    secrets:
+      gh_token: ${{ secrets.github_token }}
+
+  ymlchecks:
+    runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
-        python-version: '3.10'
+        python-version: '3.13'
     - name: Install dependencies
       run: |
-        pip install tox
-    - name: Set Tox Params
+        pip install -r requirements_dev.txt
+        pip install -e .
+    - name: Generate the YAML files
       run: |
-        echo "TOXENV=${{ matrix.jobs }}" >> $GITHUB_ENV
-    - name: Run the tests
-      env:
-        SQLFLUFF_BENCHMARK_API_KEY: ${{ secrets.SQLFLUFF_BENCHMARK_API_KEY }}
+        python test/generate_parse_fixture_yml.py
+    - name: Test the generated YAML files
       run: |
-        tox -e ${{ env.TOXENV }}
+        if [ -n "$(git status --porcelain)" ]; then
+          git diff
+          echo "Generated YAML files do not match branch."
+          echo "Please run the following command to generate these:"
+          echo "  python test/generate_parse_fixture_yml.py"
+          exit 1
+        fi
+
   examples:
     runs-on: ubuntu-latest
     name: example tests
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
-        python-version: '3.10'
+        python-version: '3.13'
     - name: Install dependencies
       run: |
         pip install -e .
@@ -125,66 +202,27 @@ jobs:
           echo "Running $file"
           python "$file"
         done
-  dbt-tests:
-    runs-on: ubuntu-latest
-    services:
-      # Label used to access the service container
-      postgres:
-        # Docker Hub image
-        image: postgres
-        # Provide the password for postgres
-        env:
-          POSTGRES_PASSWORD: password
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          # Maps tcp port 5432 on service container to the host
-          - 5432:5432
-    strategy:
-      matrix:
-        dbt-version: [ 'dbt020', 'dbt021', 'dbt100' ]
-    name: DBT Plugin ${{ matrix.dbt-version }} tests
-    steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python
-      uses: actions/setup-python@v2
-      with:
-        python-version: '3.8'
-    - name: Install dependencies
-      run: |
-        pip install tox
-    - name: Run the tests
-      run: |
-        tox -e ${{ matrix.dbt-version }} -- plugins/sqlfluff-templater-dbt
-    - name: Upload Coverage Report
-      uses: codecov/codecov-action@v1
-      with:
-        files: ./coverage.xml
+
   python-windows-tests:
     runs-on: windows-latest
-    name: Python 3.8 Windows tests
+    name: Python 3.13 Windows tests
     steps:
     - name: Set git to use LF
       run: |
         git config --global core.autocrlf false
         git config --global core.eol lf
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
-        python-version: '3.8'
+        python-version: "3.13"
     - name: List Env
       shell: bash
       run: |
         env | sort
     - name: Install dependencies
       shell: bash
-      run: |
-        pip install tox
+      run: pip install tox
     - name: Run the tests
       shell: bash
       # Set python temp dir in working dir as on GitHub Actions Windows
@@ -192,14 +230,18 @@ jobs:
       # working dir on D drive which causes problems.
       run: |
         mkdir temp_pytest
-        python -m tox -e winpy -- -n 2 test
-    - name: Upload Coverage Report
-      uses: codecov/codecov-action@v1
+        python -m tox -e winpy -- --cov=sqlfluff -n 2 test -m "not integration"
+    - name: Upload coverage data (github)
+      uses: actions/upload-artifact@v4
       with:
-        files: ./coverage.xml
+        name: coverage-data-winpy3.13
+        path: ".coverage.*"
+        if-no-files-found: ignore
+        include-hidden-files: true
+
   python-windows-dbt-tests:
     runs-on: windows-latest
-    name: DBT Plugin Python 3.8 Windows tests
+    name: dbt Plugin Python 3.12 Windows tests
     steps:
     - name: Start PostgreSQL on Windows
       run: |
@@ -213,25 +255,22 @@ jobs:
       run: |
         git config --global core.autocrlf false
         git config --global core.eol lf
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
-        python-version: '3.8'
+        # NOTE: As of 2024-10-10, dbt does not yet support python 3.13.
+        python-version: "3.12"
     - name: Install dependencies
       shell: bash
-      run: |
-        pip install tox
+      run: pip install tox
     - name: Run the tests
       shell: bash
       # Do not set explicitly temp dir for dbt as causes problems
       # None of these test need temp dir set
       run: |
-        python -m tox -e dbt100-winpy -- plugins/sqlfluff-templater-dbt
-    - name: Upload Coverage Report
-      uses: codecov/codecov-action@v1
-      with:
-        files: ./coverage.xml
+        python -m tox -e dbt180-winpy -- plugins/sqlfluff-templater-dbt
+
   pip-test-pull-request:
     # Test that using pip install works as we've missed
     # some dependencies in the past - see #1842
@@ -240,10 +279,10 @@ jobs:
     name: pip install tests
     steps:
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
-        python-version: '3.10'
-    - uses: actions/checkout@v2
+        python-version: "3.13"
+    - uses: actions/checkout@v4
     - name: Install dependencies
       run: |
         pip install .
@@ -252,13 +291,69 @@ jobs:
         sqlfluff --version
     - name: Run a simple select parse test via stdin
       run: |
-        echo "select 1" | sqlfluff parse -
+        echo "select 1" | sqlfluff parse --dialect=ansi -
     - name: Run a simple select lint test via stdin
       run: |
-        echo "select 1" | sqlfluff lint -
+        echo "select 1" | sqlfluff lint --dialect=ansi -
     - name: Run a simple select parse test via file
       run: |
-        sqlfluff parse <(echo "select 1")
+        sqlfluff parse --dialect=ansi <(echo "select 1")
     - name: Run a simple select lint test via file
       run: |
-        sqlfluff lint <(echo "select 1")
+        sqlfluff lint --dialect=ansi <(echo "select 1")
+
+  coverage_check:
+    name: Combine & check 100% coverage.
+    runs-on: ubuntu-latest
+    needs: [python-version-tests, dbt-tests, python-windows-tests, dialect-tests]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - run: python -m pip install --upgrade coverage[toml]
+
+      - name: Download coverage data.
+        uses: actions/download-artifact@v4
+        with:
+          pattern: coverage-data-*
+          merge-multiple: true
+
+      - name: Combine coverage & fail if it's <100%.
+        id: report_coverage
+        # NOTE: Setting the pipefail option here means that even when
+        # piping the output to `tee`, we still get the exit code of the
+        # `coverage report` command.
+        run: |
+          set -o pipefail
+          python -m coverage combine
+          python -m coverage html --skip-covered --skip-empty
+          python -m coverage report --fail-under=100 --skip-covered --skip-empty -m | tee coverage-report.txt
+
+      - name: Upload HTML report if check failed.
+        uses: actions/upload-artifact@v4
+        with:
+          name: html-report
+          path: htmlcov
+        if: failure() && github.event_name == 'pull_request'
+
+      - name: Stash PR Number.
+        if: always() && github.event_name == 'pull_request'
+        # NOTE: We do this so we know what PR to comment on when we pick up the report.
+        run: |
+          echo ${{ github.event.number }} > ./pr-number.txt
+          echo ${{ steps.report_coverage.outcome }} > ./outcome.txt
+
+      - name: Upload TXT report always (to add as comment to PR).
+        # NOTE: We don't actually comment on the PR from here, we'll do that in
+        # a more secure way by triggering a more secure workflow.
+        # https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
+        uses: actions/upload-artifact@v4
+        with:
+          name: txt-report
+          path: |
+            coverage-report.txt
+            pr-number.txt
+            outcome.txt
+        if: always() && github.event_name == 'pull_request'

.github/workflows/create-release-pull-request.yaml

@@ -0,0 +1,71 @@
+name: Create release pull request
+
+on:
+  workflow_dispatch:
+    inputs:
+      newVersionNumber:
+        description: 'New version number'
+        required: true
+
+jobs:
+  run:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Abort if branch already exists
+        run: |
+            _check_branch=$(git ls-remote --heads origin prep-${{ github.event.inputs.newVersionNumber }})
+            if [[ -z ${_check_branch} ]]; then
+                echo "Release branch doesn't exist yet, continuing"
+            else
+                echo "Release branch already exists, aborting. Run the Python release script locally."
+                exit 1
+            fi
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          pip install requests click pyyaml ghapi
+
+      - name: Prepare release
+        run: |
+          python util.py release ${{ github.event.inputs.newVersionNumber }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_REPOSITORY_OWNER: ${{ secrets.GITHUB_REPOSITORY_OWNER }}
+
+      - name: Create pull request
+        uses: peter-evans/create-pull-request@v4
+        with:
+          delete-branch: true
+          branch: prep-${{ github.event.inputs.newVersionNumber }}
+          commit-message: "Bump to version ${{ github.event.inputs.newVersionNumber }}"
+          title: Prep version ${{ github.event.inputs.newVersionNumber }}
+          body: |
+            Prepare version ${{ github.event.inputs.newVersionNumber }}
+
+            Please add all merged changes from [Release Draft][1] to
+            the CHANGELOG.md file
+
+            - Auto-generated by [create-pull-request][2] GitHub Action
+
+            [1]: https://github.com/sqlfluff/sqlfluff/releases
+            [2]: https://github.com/peter-evans/create-pull-request
+          labels: |
+            release
+            skip-changelog
+
+      - name: Update release title and tag
+        uses: release-drafter/release-drafter@v5
+        with:
+          # NOTE: We should eventually actually populate the date here, but that
+          # will most likely change before the new pull request actually gets
+          # merged, so we just add "YYYY-MM-DD" for now as a placeholder.
+          name: "[${{ github.event.inputs.newVersionNumber }}] - YYYY-MM-DD"
+          tag: ${{ github.event.inputs.newVersionNumber }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/pre-commit.yml

@@ -0,0 +1,53 @@
+---
+name: pre-commit
+on:
+  pull_request:
+  push:
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    env:
+      RAW_LOG: pre-commit.log
+      CS_XML: pre-commit.xml
+      SKIP: no-commit-to-branch
+    steps:
+      - run: sudo apt-get update && sudo apt-get install cppcheck
+        if: false
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        if: false
+        with:
+          cache: pip
+          python-version: 3.12.1
+      - run: python -m pip install pre-commit
+      - uses: actions/cache/restore@v4
+        with:
+          path: ~/.cache/pre-commit/
+          key: pre-commit-4|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml')
+            }}
+      - name: Run pre-commit hooks
+        run: |
+          set -o pipefail
+          pre-commit gc
+          pre-commit run --show-diff-on-failure --color=always --all-files | tee ${RAW_LOG}
+      - name: Convert Raw Log to Checkstyle format (launch action)
+        uses: mdeweerd/logToCheckStyle@v2024.3.5
+        if: ${{ failure() }}
+        with:
+          in: ${{ env.RAW_LOG }}
+          out: ${{ env.CS_XML }}
+      - uses: actions/cache/save@v4
+        if: ${{ ! cancelled() }}
+        with:
+          path: ~/.cache/pre-commit/
+          key: pre-commit-4|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml')
+            }}
+      - name: Provide log as artifact
+        uses: actions/upload-artifact@v4
+        if: ${{ ! cancelled() }}
+        with:
+          name: precommit-logs
+          path: |
+            ${{ env.RAW_LOG }}
+            ${{ env.CS_XML }}
+          retention-days: 2

.github/workflows/publish-dbt-templater-release-to-pypi.yaml

@@ -10,10 +10,10 @@ jobs:
   run:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v1
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
-          python-version: "3.7"
+          python-version: "3.10"
 
       - name: Install Dependencies
         run: |
@@ -28,8 +28,8 @@ jobs:
         run: cp -r plugins/sqlfluff-templater-dbt/dist/. dist/
 
       - name: Publish Python distribution to PyPI
-        uses: pypa/gh-action-pypi-publish@master
+        uses: pypa/gh-action-pypi-publish@release/v1
         with:
           user: __token__
           password: ${{ secrets.PYPI_DBT_TEMPLATER_TOKEN }}
-          skip_existing: true
+          skip-existing: true

.github/workflows/publish-sqlfluff-docker-image-to-dockerhub.yaml

@@ -25,15 +25,15 @@ jobs:
       # Setup QEMU and Buildx to allow for multi-platform builds.
       - name: Set up QEMU
         id: docker_qemu
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
         id: docker_buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
 
       # Authenticate with DockerHub.
       - name: Login to DockerHub
         id: docker_login
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -41,7 +41,7 @@ jobs:
       # Build amd64 image to use in the integration test.
       - name: Build and export to Docker
         id: docker_build
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v5
         with:
           load: true
           tags: ${{ env.TEST_TAG }}
@@ -53,13 +53,13 @@ jobs:
         id: docker_test
         run: |
           echo "SELECT 1" > test.sql
-          docker run --rm -i -v $PWD:/sql ${{ env.TEST_TAG }} lint /sql/test.sql
+          docker run --rm -i -v $PWD:/sql ${{ env.TEST_TAG }} lint --dialect ansi /sql/test.sql
 
       # Build arm64 image (amd64 is cached from docker_build step) and export to DockerHub.
       # N.B. We tag this image as both latest and with its version number.
       - name: Build and push
         id: docker_build_push
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v5
         with:
           push: true
           platforms: linux/amd64,linux/arm64

.github/workflows/publish-sqlfluff-release-to-pypi.yaml

@@ -10,10 +10,10 @@ jobs:
   run:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v1
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
-          python-version: "3.7"
+          python-version: "3.10"
 
       - name: Install Dependencies
         run: |
@@ -23,8 +23,8 @@ jobs:
         run: tox -e build-dist
 
       - name: Publish Python distribution to PyPI
-        uses: pypa/gh-action-pypi-publish@master
+        uses: pypa/gh-action-pypi-publish@release/v1
         with:
           user: __token__
           password: ${{ secrets.PYPI_TOKEN }}
-          skip_existing: true
+          skip-existing: true

.gitignore

@@ -13,6 +13,7 @@ build
 _build
 dist
 .pytest_cache
+/sqlfluff-*
 
 # Ignore the Environment
 env
@@ -34,15 +35,30 @@ test-reports
 # Ignore root testing sql & python files
 /test*.sql
 /test*.py
+/test*.txt
 /.hypothesis/
 
 # Ignore dbt outputs from testing
 /target
 
+# Ignore any timing outputs
+/*.csv
+
 # Ignore conda environment.yml contributors might be using and direnv config
 environment.yml
 .envrc
 **/*FIXED.sql
-
+*.prof
 # Ignore temp packages.yml generated during testing.
 plugins/sqlfluff-templater-dbt/test/fixtures/dbt/dbt_project/packages.yml
+
+# VSCode
+.vscode
+*.code-workspace
+
+# Emacs
+*~
+
+# Mypyc outputs
+*.pyd
+*.so

.pre-commit-config.yaml

@@ -1,49 +1,95 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.1.0
+    rev: v4.5.0
     hooks:
+      - id: no-commit-to-branch
+        args: [--branch, main]
+      # If adding any exceptions here, make sure to add them to .editorconfig as well
       - id: end-of-file-fixer
-        exclude: (.*\.sql$|test/fixtures/linter/sqlfluffignore/path_c)
+        exclude: |
+          (?x)^
+          (
+            test/fixtures/templater/jinja_l_metas/0(0[134578]|11).sql|
+            test/fixtures/linter/sqlfluffignore/[^/]*/[^/]*.sql|
+            test/fixtures/config/inheritance_b/(nested/)?example.sql|
+            (.*)/trailing_newlines.sql|
+            plugins/sqlfluff-templater-dbt/test/fixtures/dbt.*/dbt_project/models/my_new_project/multiple_trailing_newline.sql|
+            plugins/sqlfluff-templater-dbt/test/fixtures/dbt.*/templated_output/macro_in_macro.sql|
+            plugins/sqlfluff-templater-dbt/test/fixtures/dbt.*/templated_output/(dbt_utils_0.8.0/)?last_day.sql|
+            test/fixtures/linter/indentation_errors.sql|
+            test/fixtures/templater/jinja_d_roundtrip/test.sql
+          )$
       - id: trailing-whitespace
-        exclude: (.*\.sql$|test/fixtures/linter/sqlfluffignore/path_c)
+        exclude: |
+          (?x)^(
+            test/fixtures/linter/indentation_errors.sql|
+            test/fixtures/templater/jinja_d_roundtrip/test.sql|
+            test/fixtures/config/inheritance_b/example.sql|
+            test/fixtures/config/inheritance_b/nested/example.sql|
+            plugins/sqlfluff-templater-dbt/test/fixtures/dbt.*/templated_output/macro_in_macro.sql|
+            plugins/sqlfluff-templater-dbt/test/fixtures/dbt.*/templated_output/last_day.sql|
+            plugins/sqlfluff-templater-dbt/test/fixtures/dbt.*/templated_output/dbt_utils_0.8.0/last_day.sql|
+            test/fixtures/linter/sqlfluffignore/
+          )$
   - repo: https://github.com/psf/black
-    rev: 22.1.0
+    rev: 24.2.0
     hooks:
       - id: black
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.931
+    rev: v1.13.0
     hooks:
       - id: mypy
-        args: [--ignore-missing-imports]
-        additional_dependencies: [
-          types-toml,
-          types-pkg_resources,
-          types-chardet,
-          types-appdirs,
-          types-colorama,
-          types-pyyaml,
-          types-regex,
-        ]
+        additional_dependencies:
+          # NOTE: These dependencies should be the same as the `types-*` dependencies in
+          # `requirements_dev.txt`. If you update these, make sure to update those too.
+          [
+            types-toml,
+            types-chardet,
+            types-colorama,
+            types-pyyaml,
+            types-regex,
+            types-tqdm,
+            # Type stubs are obvious to import, but some dependencies also define their own
+            # types directly (e.g. jinja). pre-commit doesn't actually install the python
+            # package, and so doesn't automatically install the dependencies from
+            # `pyproject.toml` either. We include them here to make sure mypy can function
+            # properly.
+            jinja2,
+            pathspec,
+            pytest, # and by extension... pluggy
+            click,
+            platformdirs
+          ]
         files: ^src/sqlfluff/.*
+        # The mypy pre-commit hook by default sets a few arguments that we don't normally
+        # use. To undo that we reset the `args` to be empty here. This is important to
+        # ensure we don't get conflicting  results from the pre-commit hook and from the
+        # CI job.
+        args: []
   - repo: https://github.com/pycqa/flake8
-    rev: 4.0.1
+    rev: 7.0.0
     hooks:
       - id: flake8
-        additional_dependencies: [
-          flake8-black>=0.2.4,
-          flake8-docstrings,
-        ]
+        additional_dependencies: [flake8-black>=0.3.6]
   - repo: https://github.com/pycqa/doc8
-    rev: 0.10.1
+    rev: v1.1.1
     hooks:
       - id: doc8
-        args: [
-          --file-encoding,
-          utf8,
-        ]
+        args: [--file-encoding, utf8]
         files: docs/source/.*\.rst$
   - repo: https://github.com/adrienverge/yamllint.git
-    rev: v1.26.3
+    rev: v1.35.1
     hooks:
       - id: yamllint
         args: [-c=.yamllint]
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    # Ruff version.
+    rev: "v0.3.2"
+    hooks:
+      - id: ruff
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.2.6
+    hooks:
+      - id: codespell
+        exclude: (?x)^(test/fixtures/.*|pyproject.toml)$
+        additional_dependencies: [tomli]

.pre-commit-hooks.yaml

@@ -1,19 +1,26 @@
 - id: sqlfluff-lint
   name: sqlfluff-lint
-  entry: sqlfluff lint
+  # Set `--processes 0` to use maximum parallelism
+  #  - `--disable-progress-bar` pre-commit suppresses logging already
+  #      this can cause an unneeded slow down.
+  entry: sqlfluff lint --processes 0 --disable-progress-bar
   language: python
-  description: 'Lints sql files with `SQLFluff`'
+  description: "Lints sql files with `SQLFluff`"
   types: [sql]
   require_serial: true
   additional_dependencies: []
 
 - id: sqlfluff-fix
   name: sqlfluff-fix
-  # Needs to use "--force" to disable confirmation
-  # By default all the rules are applied
-  entry: sqlfluff fix --force
+  # Set a couple of default flags:
+  #  - `--show-lint-violations` shows issues to not require running `sqlfluff lint`
+  #  - `--processes 0` to use maximum parallelism
+  #  - `--disable-progress-bar` pre-commit suppresses logging already
+  #      this can cause an unneeded slow down.
+  # By default, this hook applies all rules.
+  entry: sqlfluff fix --show-lint-violations --processes 0 --disable-progress-bar
   language: python
-  description: 'Fixes sql lint errors with `SQLFluff`'
+  description: "Fixes sql lint errors with `SQLFluff`"
   types: [sql]
   require_serial: true
   additional_dependencies: []

.readthedocs.yml

@@ -15,9 +15,16 @@ formats: []
 # Optionally set the version of Python and requirements required to build your docs.
 # In our case we need both the docs requirements and the package itself.
 python:
-  version: 3.7
   install:
-    - requirements: requirements.txt
     - requirements: docs/requirements.txt
-    - method: setuptools
+    - method: pip
       path: .
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+  jobs:
+    # Before building, generate the rule & dialect docs
+    pre_build:
+      - python docs/generate-auto-docs.py

CODE_OF_CONDUCT.md

@@ -6,7 +6,7 @@
 We as members, contributors, and leaders pledge to make participation in our
 community a harassment-free experience for everyone, regardless of age, body
 size, visible or invisible disability, ethnicity, sex characteristics, gender
-identity and expression, level of experience, education, socio-economic status,
+identity and expression, level of experience, education, socioeconomic status,
 nationality, personal appearance, race, religion, or sexual identity
 and orientation.
 

CONTRIBUTING.md

@@ -25,12 +25,11 @@ for the wiki.
 
 :star2: **Fifth** - if you are so inclined - pull requests on the core codebase
 are always welcome. Dialect additions are often a good entry point for new
-contributors, and we have
-[a wiki page](https://github.com/sqlfluff/sqlfluff/wiki/Contributing-Dialect-Changes)
+contributors, and we have [a set of guides](https://docs.sqlfluff.com/en/stable/perma/guides.html)
 to help you through your first contribution. Bear in mind that all the tests
 should pass, and test coverage should not decrease unduly as part of the
 changes which you make. You may find it useful to familiarise yourself with the
-[architectural principles here](https://docs.sqlfluff.com/en/latest/architecture.html)
+[architectural principles here](https://docs.sqlfluff.com/en/stable/perma/architecture.html)
 and with the [current documentation here](https://docs.sqlfluff.com).
 
 ## How The Community Works
@@ -82,13 +81,25 @@ changes.
 
 ### Developing and Running SQLFluff Locally
 
-The simplest way to set up a development environment is to use `tox`.
-First ensure that you have tox installed (windows users may have to replace `python3` with `py`):
-```shell
-python3 -m pip install -U tox
-```
+#### Requirements
 
-A virtual environment can then be created and activated by running:
+The simplest way to set up a development environment is to use `tox`.
+First ensure that you have tox installed:
+```shell
+python3.8 -m pip install -U tox
+```
+**IMPORTANT:** `tox` must be installed with a minimum of Python 3.8 as
+the `mypy` checks are incompatible with 3.7. Those using newer versions of
+Python may replace `python3.8` as necessary (the test suite runs primarily
+under 3.12 for example).
+
+Note: Unfortunately tox does not currently support setting just a minimum
+Python version (though this may be be coming in tox 4!).
+
+#### Creating a virtual environment
+
+A virtual environment can then be created and activated by running (check
+the [requirements](#requirements) before running this):
 ```shell
 tox -e dbt021-py38 --devenv .venv
 source .venv/bin/activate
@@ -102,14 +113,15 @@ choose one of the dbt environments.)
 
 Windows users should call `.venv\Scripts\activate` rather than `source .venv/bin/activate`.
 
-This virtual environment will already have the package installed in editable mode for you, as well as
-`requirements_dev.txt` and `plugins/sqlfluff-plugin-example`. Additionally if a dbt virtual environment
-was specified, you will also have `dbt-core`, `dbt-postgres`, and `plugins/sqlfluff-templater-dbt` available.
+This virtual environment will already have the package installed in editable
+mode for you, as well as `requirements_dev.txt` and `plugins/sqlfluff-plugin-example`.
+Additionally if a dbt virtual environment was specified, you will also have
+`dbt-core`, `dbt-postgres`, and `plugins/sqlfluff-templater-dbt` available.
 
 ### Wiki
 
 We have a [GitHub wiki](https://github.com/sqlfluff/sqlfluff/wiki) with some
-more long form tutorials for contributors, particualrly those new to SQLFluff
+more long form tutorials for contributors, particularly those new to SQLFluff
 or contributing to open source. We welcome
 [contributions, suggestions or requests](https://github.com/sqlfluff/sqlfluff/issues/2104)
 for the wiki.
@@ -124,13 +136,14 @@ but you'll need to do each one explicitly. e.g.
 pip install -e plugins/sqlfluff-templater-dbt/.
 ```
 
-> NOTE: For packages intended to be installed like this, the source code must be directy
+> NOTE: For packages intended to be installed like this, the source code must be directly
 > within a subdirectory with the name of the package and not in a subdirectory such as
 > src. This is due to a restriction in the implementation of setup.py in editable mode.
 
 ### Testing
 
-To test locally, SQLFluff uses `tox`. The test suite can be run via:
+To test locally, SQLFluff uses `tox` (check the [requirements](#requirements)!).
+The test suite can be run via:
 
 ```shell
 tox
@@ -160,7 +173,7 @@ faster while working on an issue, before running full tests at the end.
 For example, you can run specific tests by making use of the `-k` option in `pytest`:
 
 ```
-tox -e py38 -- -k L012 test
+tox -e py38 -- -k AL02 test
 ```
 
 Alternatively, you can also run tests from a specific directory or file only:
@@ -175,9 +188,67 @@ sqlfluff parse test.sql
 ```
 (ensure your virtual environment is activated first).
 
+#### How to use and understand the test suite
+
+When developing for SQLFluff, you may not need (or wish) to run the whole test
+suite, depending on what you are working on. Here are a couple of scenarios
+for development, and which parts of the test suite you may find most useful.
+
+1. For dialect improvements (i.e. changes to anything in [src/sqlfluff/dialects](./src/sqlfluff/dialects))
+   you should not need to continuously run the full core test suite. Running
+   either `tox -e generate-fixture-yml` (if using tox), or setting up a python
+   virtualenv and running `test/generate_parse_fixture_yml.py` directly will
+   usually be sufficient. Both of these options accept arguments to restrict
+   runs to specific dialects to further improve iteration speed. e.g.
+   - `tox -e generate-fixture-yml -- -d mysql` will run just the mysql tests.
+   - `python test/generate_parse_fixture_yml.py -d mysql` will do the same.
+2. Developing for the dbt templater should only require running the dbt test
+   suite (see below).
+3. Developing rules and rule plugins there are a couple of scenarios.
+   - When developing a new rule or working with a more isolated rule, you
+     should only need to run the tests for that rule. These are usually what
+     are called the _yaml tests_. This refers to a body of example sql
+     statements and potential fixes defined in a large set of yaml files
+     found in [test/fixtures/rules/std_rule_cases](./test/fixtures/rules/std_rule_cases).
+     The easiest way to run these is by calling that part of the suite
+     directly and filtering to just that rule. For example:
+     - `tox -e py39 -- test/rules/yaml_test_cases_test.py -k AL01`
+     - `pytest test/rules/yaml_test_cases_test.py -k AL01`
+   - When developing on some more complicated rules, or ones known to
+     have interactions with other rules, there are a set of rule fixing
+     tests which apply a set combination of those rules. These are best
+     run via the `autofix` tests. For example:
+     - `tox -e py39 -- test/rules/std_fix_auto_test.py`
+     - `pytest test/rules/std_fix_auto_test.py`
+     - Potentially even the full rules suite `tox -e py39 -- test/rules`
+   - A small number of core rules are also used in making sure that inner
+     parts of SQLFluff are also functioning. This isn't great isolation
+     but does mean that occasionally you may find side effects of your
+     changes in the wider test suite. These can usually be caught by
+     running the full `tox -e py39` suite as a final check (or using the
+     test suite on GitHub when posting your PR).
+4. When developing the internals of SQLFluff (i.e. anything not
+   already mentioned above), the test suite typically mirrors the structure
+   of the internal submodules of sqlfluff:
+   - When working with the CLI, the `sqlfluff.cli` module has a test suite
+     called via `tox -e py39 -- test/cli`.
+   - When working with the templaters (i.e. `sqlfluff.core.templaters`), the
+     corresponding test suite is found via `tox -e py39 -- test/core/templaters`.
+   - This rough guidance and may however not apply for all of the internals.
+     For example, changes to the internals of the parsing module (`sqlfluff.core.parser`)
+     are very likely to have knock-on implications across the rest of the test
+     suite and it may be necessary to run the whole thing. In these
+     situations however you can usually work slowly outward, for example:
+     1. If your change is to the `AnyOf()` grammar, first running `tox -e py39 -- test/core/parser/grammar_test.py` would be wise.
+     2. ...followed by `tox -e py39 -- test/core/parser` once the above is passing.
+     3. ...and then `tox -e py39 -- test/core`.
+     4. ...and finally the full suite `tox -e py39`.
+
 #### dbt templater tests
 
-The dbt templater tests require a locally running Postgres instance. See the required connection parameters in `plugins/sqlfluff-templater-dbt/test/fixtures/dbt/profiles.yml`. We recommend using https://postgresapp.com/.
+The dbt templater tests require a locally running Postgres instance. See the
+required connection parameters in `plugins/sqlfluff-templater-dbt/test/fixtures/dbt/profiles.yml`.
+We recommend using https://postgresapp.com/.
 
 To run the dbt-related tests you will have to explicitly include these tests:
 
@@ -199,8 +270,8 @@ Docker Compose environment. It's a simple two-container configuration:
 Steps to use the Docker Compose environment:
 * Install Docker on your machine.
 * Run `plugins/sqlfluff-templater-dbt/docker/startup` to create the containers.
-* Run `plugins/sqlfluff-templater-dbt/docker/shell` to start a bash session in the `app` container.
-* Manually edit the `host` value in `plugins/sqlfluff-templater-dbt/test/fixtures/dbt/profiles_yml/profiles.yml`, changing it to `postgres`. (This will likely be automated somehow in the future.)
+* Run `plugins/sqlfluff-templater-dbt/docker/shell` to start a bash session
+  in the `app` container.
 
 Inside the container, run:
 ```
@@ -209,12 +280,23 @@ py.test -v plugins/sqlfluff-templater-dbt/test/
 
 ### Pre-Commit Config
 
-For development convenience we also provide a `.pre-commit-config.yaml` file to allow the user to install a selection of pre-commit hooks via `tox -e pre-commit -- install`. These hooks can help the user identify and fix potential linting/typing violations prior to committing their code and therefore reduce having to deal with these sort of issues during code review.
+For development convenience we also provide a `.pre-commit-config.yaml` file
+to allow the user to install a selection of pre-commit hooks by running (check
+the [requirements](#requirements) before running this):
+
+```
+tox -e pre-commit -- install
+```
+
+These hooks can help the user identify and fix potential linting/typing
+violations prior to committing their code and therefore reduce having to deal
+with these sort of issues during code review.
 
 ### Documentation Website
 
-Documentation is built using Sphinx with some pages being built based on the source code.
-See the [Documentation Website README.md](./docs/README.md) file for more information on how to build and test this.
+Documentation is built using Sphinx with some pages being built based on the
+source code. See the [Documentation Website README.md](./docs/README.md) file
+for more information on how to build and test this.
 
 ### Building Package
 
@@ -224,26 +306,81 @@ whenever a new release is published to GitHub.
 
 #### Release checklist:
 
-The [release page](https://github.com/sqlfluff/sqlfluff/releases) shows maintainers all merges since last release. Once we have a long enough list, we should prepare a release, following below checklist:
+The [release page](https://github.com/sqlfluff/sqlfluff/releases) shows
+maintainers all merges since last release. Once we have a long enough list,
+we should prepare a release.
 
-- [ ] Change the version in `setup.cfg` and `plugins/sqlfluff-templator-dbt/setup.cfg`
+A release PR can be created by maintainers via the
+["Create release pull request" GitHub Action](https://github.com/sqlfluff/sqlfluff/actions/workflows/create-release-pull-request.yaml).
+
+As further PRs are merged, we may need to rerun the release script again
+(or alternatively just manually updating the branch). This can only be rerun
+locally (the GitHub Action will exit error if the branch already exists to
+prevent overwriting it).
+
+Check out the release branch created by the GitHub Action locally and run
+the script. It will preserve any `Highlights` you have added and update the
+other sections with new contributions. It can be run as follows (you will
+need a [GitHub Personal Access Token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) with "repo" permission):
+
+```shell
+source .venv/bin/activate
+export GITHUB_REPOSITORY_OWNER=sqlfluff
+export GITHUB_TOKEN=gho_xxxxxxxx # Change to your token with "repo" permissions.
+python util.py release 2.0.3 # Change to your release number
+```
+
+Below is the old list of release steps, but many are automated by the process
+described above.
+
+- [ ] Change the version in `setup.cfg` and `plugins/sqlfluff-templater-dbt/setup.cfg`
 - [ ] Update the stable_version in the `[sqlfluff_docs]` section of `setup.cfg`
-- [ ] Copy the draft releases from https://github.com/sqlfluff/sqlfluff/releases to [CHANGELOG.md](CHANGELOG.md). These draft release notes have been created by a GitHub Action on each PR merge.
-- [ ] If you pretend to create a new draft in GitHub and hit "Auto Generate Release Notes", then it will basically recreate these notes (though in a slightly different format), but also add a nice "First contributors" section, so can copy that "First contributors" section too and then abandon that new draft ([an issues](https://github.com/release-drafter/release-drafter/issues/1001) has been raised to ask for this in Release Drafter GitHub Action).
-- [ ] Add markdown links to PRs as annoyingly GitHub doesn't do this automatically when displaying Markdown files, like it does for comments. You can use regex in most code editors to replace `\(#([0-9]*)\) @([^ ]*)$` to `[#$1](https://github.com/sqlfluff/sqlfluff/pull/$1) [@$2](https://github.com/$2)`, or if using the GitHub generated release notes then can replace `by @([^ ]*) in https://github.com/sqlfluff/sqlfluff/pull/([0-9]*)$` to `[#$2](https://github.com/sqlfluff/sqlfluff/pull/$2) [@$1](https://github.com/$1)`.
-- [ ] For the new contributors section, you can replace `\* @([^ ]*) made their first contribution in https://github.com/sqlfluff/sqlfluff/pull/([0-9]*)$` with `* [@$1](https://github.com/$1) made their first contribution in [#$2](https://github.com/sqlfluff/sqlfluff/pull/$2)` to do this automatically).
-- [ ] Check each issue title is clear, and if not edit issue title (which will automatically update Release notes on next PR merged, as the Draft one is recreated in full each time). We also don't use [conventional commit PR titles](https://www.conventionalcommits.org/en/v1.0.0/) (e.g. `feat`) so make them more English readible. Make same edits locally in [CHANGELOG.md](CHANGELOG.md).
+- [ ] Copy the draft releases from https://github.com/sqlfluff/sqlfluff/releases
+      to [CHANGELOG.md](CHANGELOG.md). These draft release notes have been created
+      by a GitHub Action on each PR merge.
+- [ ] If you pretend to create a new draft in GitHub and hit "Auto Generate Release
+      Notes", then it will basically recreate these notes (though in a slightly
+      different format), but also add a nice "First contributors" section, so can
+      copy that "First contributors" section too and then abandon that new draft
+      ([an issues](https://github.com/release-drafter/release-drafter/issues/1001)
+      has been raised to ask for this in Release Drafter GitHub Action).
+- [ ] Add markdown links to PRs as annoyingly GitHub doesn't do this automatically
+      when displaying Markdown files, like it does for comments. You can use regex
+      in most code editors to replace `\(#([0-9]*)\) @([^ ]*)$` to
+      `[#$1](https://github.com/sqlfluff/sqlfluff/pull/$1) [@$2](https://github.com/$2)`,
+      or if using the GitHub generated release notes then can replace
+      `by @([^ ]*) in https://github.com/sqlfluff/sqlfluff/pull/([0-9]*)$` to
+      `[#$2](https://github.com/sqlfluff/sqlfluff/pull/$2) [@$1](https://github.com/$1)`.
+- [ ] For the new contributors section, you can replace
+      `\* @([^ ]*) made their first contribution in https://github.com/sqlfluff/sqlfluff/pull/([0-9]*)$`
+      with `* [@$1](https://github.com/$1) made their first contribution in [#$2](https://github.com/sqlfluff/sqlfluff/pull/$2)` to do this automatically).
+- [ ] Check each issue title is clear, and if not edit issue title (which will
+      automatically update Release notes on next PR merged, as the Draft one is
+      recreated in full each time). We also don't use
+      [conventional commit PR titles](https://www.conventionalcommits.org/en/v1.0.0/)
+      (e.g. `feat`) so make them more English readable. Make same edits locally
+      in [CHANGELOG.md](CHANGELOG.md).
 - [ ] Add a comment at the top to highlight the main things in this release.
-- [ ] If this is a non-patch release then update the `Notable changes` section in `index.rst` with a brief summary of the new features added that made this a non-patch release.
-- [ ] View the CHANGELOG in this branch on GitHub to ensure you didn't miss any link conversions or other markup errors.
-- [ ] Open draft PR with those change a few days in advance to give contributors notice. Tag those with open PRs in the PR in GitHub to give them time to merge their work before the new release
+- [ ] If this is a non-patch release then update the `Notable changes` section in
+      `index.rst` with a brief summary of the new features added that made this a
+      non-patch release.
+- [ ] View the CHANGELOG in this branch on GitHub to ensure you didn't miss any
+      link conversions or other markup errors.
+- [ ] Open draft PR with those change a few days in advance to give contributors
+      notice. Tag those with open PRs in the PR in GitHub to give them time to merge
+      their work before the new release
 - [ ] Comment in #contributing slack channel about release candidate.
 - [ ] Update the draft PR as more changes get merged.
 - [ ] Get another contributor to approve the PR.
-- [ ] Merge the PR when looks like we've got all we’re gonna get for this release.
-- [ ] Go to the [releases page](https://github.com/sqlfluff/sqlfluff/releases), edit the release to be same as [CHANGELOG.md](CHANGELOG.md) (remember to remove your release PR which doesn’t need to go in this). Add version tag and a title and click “Publish release”.
-- [ ] Announce the release in the #general channel, with shout outs to those who contributed many, or big items.
-- [ ] Announce the release on Twitter (@tunetheweb can do this or let him know your Twitter handle if you want access to Tweet on SQLFluff’s behalf).
+- [ ] Merge the PR when looks like we've got all we’re going to get for this release.
+- [ ] Go to the [releases page](https://github.com/sqlfluff/sqlfluff/releases), edit
+      the release to be same as [CHANGELOG.md](CHANGELOG.md) (remember to remove your
+      release PR which doesn’t need to go in this). Add version tag and a title and
+      click “Publish release”.
+- [ ] Announce the release in the #general channel, with shout outs to those who
+      contributed many, or big items.
+- [ ] Announce the release on Twitter (@tunetheweb can do this or let him know your
+      Twitter handle if you want access to Tweet on SQLFluff’s behalf).
 
 :warning: **Before creating a new release, ensure that
 [setup.cfg](setup.cfg) is up-to-date with a new version** :warning:.

Dockerfile

@@ -7,23 +7,24 @@ WORKDIR /app
 ENV VIRTUAL_ENV /app/.venv
 RUN python -m venv $VIRTUAL_ENV
 ENV PATH $VIRTUAL_ENV/bin:$PATH
-RUN pip install --upgrade pip setuptools wheel
+RUN pip install --no-cache-dir --upgrade pip setuptools wheel pip-tools
 
-# Install requirements seperately
+# Install requirements separately
 # to take advantage of layer caching.
-# N.B. we extract the requirements from setup.cfg
-COPY setup.cfg .
-RUN python -c "import configparser; c = configparser.ConfigParser(); c.read('setup.cfg'); print(c['options']['install_requires'])" > requirements.txt
-RUN pip install --upgrade -r requirements.txt
+# N.B. we extract the requirements from pyproject.toml
+COPY pyproject.toml .
+# Use piptools to extract requirements from pyproject.toml as described in
+# https://github.com/pypa/pip/issues/11584
+RUN pip-compile -o requirements.txt pyproject.toml -v --strip-extras
+RUN pip install --no-cache-dir --upgrade -r requirements.txt
 
 # Copy minimal set of SQLFluff package files.
 COPY MANIFEST.in .
 COPY README.md .
-COPY setup.py .
 COPY src ./src
 
 # Install sqlfluff package.
-RUN pip install --no-dependencies .
+RUN pip install --no-cache-dir --no-dependencies .
 
 # Switch to non-root user.
 USER 5000

LICENSE.md

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2018 Alan Cruickshank
+Copyright (c) 2023 Alan Cruickshank
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -8,45 +8,96 @@
 [![PyPi Status](https://img.shields.io/pypi/status/sqlfluff.svg?style=flat-square)](https://pypi.org/project/sqlfluff/)
 [![PyPi Downloads](https://img.shields.io/pypi/dm/sqlfluff?style=flat-square)](https://pypi.org/project/sqlfluff/)
 
-[![codecov](https://img.shields.io/codecov/c/gh/sqlfluff/sqlfluff.svg?style=flat-square&logo=Codecov)](https://codecov.io/gh/sqlfluff/sqlfluff)
-[![CI Tests](https://github.com/sqlfluff/sqlfluff/workflows/CI%20Tests/badge.svg)](https://github.com/sqlfluff/sqlfluff/actions?query=workflow%3A%22CI+Tests%22)
+[![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/sqlfluff/sqlfluff/.github/workflows/ci-tests.yml?logo=github&style=flat-square)](https://github.com/sqlfluff/sqlfluff/actions/workflows/ci-tests.yml?query=branch%3Amain)
 [![ReadTheDocs](https://img.shields.io/readthedocs/sqlfluff?style=flat-square&logo=Read%20the%20Docs)](https://sqlfluff.readthedocs.io)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg?style=flat-square)](https://github.com/psf/black)
+[![Docker Pulls](https://img.shields.io/docker/pulls/sqlfluff/sqlfluff?logo=docker&style=flat-square)](https://hub.docker.com/r/sqlfluff/sqlfluff)
+[![Gurubase](https://img.shields.io/badge/Gurubase-Ask%20SQLFluff%20Guru-006BFF?style=flat-square)](https://gurubase.io/g/sqlfluff)
 
-**SQLFluff** is a dialect-flexible and configurable SQL linter. Designed with ELT applications in mind, **SQLFluff** also works with Jinja templating and dbt. **SQLFluff** will auto-fix most linting errors, allowing you to focus your time on what matters.
+**SQLFluff** is a dialect-flexible and configurable SQL linter. Designed
+with ELT applications in mind, **SQLFluff** also works with Jinja templating
+and dbt. **SQLFluff** will auto-fix most linting errors, allowing you to focus
+your time on what matters.
+
+## Table of Contents
+
+1. [Dialects Supported](#dialects-supported)
+2. [Templates Supported](#templates-supported)
+3. [VS Code Extension](#vs-code-extension)
+4. [Getting Started](#getting-started)
+5. [Documentation](#documentation)
+6. [Releases](#releases)
+7. [SQLFluff on Slack](#sqlfluff-on-slack)
+8. [SQLFluff on Twitter](#sqlfluff-on-twitter)
+9. [Contributing](#contributing)
+10. [Sponsors](#sponsors)
 
 ## Dialects Supported
 
-Although SQL is reasonably consistent in its implementations, there are several different dialects available with variations of syntax and grammar. **SQLFluff** currently supports the following SQL dialects (though perhaps not in full):
+Although SQL is reasonably consistent in its implementations, there are several
+different dialects available with variations of syntax and grammar. **SQLFluff**
+currently supports the following SQL dialects (though perhaps not in full):
 
-- ANSI SQL - this is the base version and on occasion may not strictly follow the ANSI/ISO SQL definition
+- ANSI SQL - this is the base version and on occasion may not strictly follow
+  the ANSI/ISO SQL definition
+- [Athena](https://aws.amazon.com/athena/)
 - [BigQuery](https://cloud.google.com/bigquery/)
+- [ClickHouse](https://clickhouse.com/)
+- [Databricks](https://databricks.com/) (note: this extends the `sparksql` dialect with
+  [Unity Catalog](https://docs.databricks.com/data-governance/unity-catalog/index.html) syntax).
+- [Db2](https://www.ibm.com/analytics/db2)
+- [DuckDB](https://duckdb.org/)
 - [Exasol](https://www.exasol.com/)
+- [Greenplum](https://greenplum.org/)
 - [Hive](https://hive.apache.org/)
+- [Impala](https://impala.apache.org/)
+- [Materialize](https://materialize.com/)
 - [MySQL](https://www.mysql.com/)
 - [Oracle](https://docs.oracle.com/en/database/oracle/oracle-database/21/sqlrf/index.html)
 - [PostgreSQL](https://www.postgresql.org/) (aka Postgres)
 - [Redshift](https://docs.aws.amazon.com/redshift/index.html)
 - [Snowflake](https://www.snowflake.com/)
-- [Spark3](https://spark.apache.org/docs/latest/)
+- [SOQL](https://developer.salesforce.com/docs/atlas.en-us.soql_sosl.meta/soql_sosl/sforce_api_calls_soql.htm)
+- [SparkSQL](https://spark.apache.org/docs/latest/)
 - [SQLite](https://www.sqlite.org/)
+- [StarRocks](https://www.starrocks.io)
 - [Teradata](https://www.teradata.com/)
 - [Transact-SQL](https://docs.microsoft.com/en-us/sql/t-sql/language-reference) (aka T-SQL)
+- [Trino](https://trino.io/)
+- [Vertica](https://www.vertica.com/)
 
-We aim to make it easy to expand on the support of these dialects and also add other, currently unsupported, dialects. Please [raise issues](https://github.com/sqlfluff/sqlfluff/issues) (or upvote any existing issues) to let us know of demand for missing support.
+We aim to make it easy to expand on the support of these dialects and also
+add other, currently unsupported, dialects. Please [raise issues](https://github.com/sqlfluff/sqlfluff/issues)
+(or upvote any existing issues) to let us know of demand for missing support.
 
-Pull requests from those that know the missing syntax or dialects are especially welcomed and are the question way for you to get support added. We are happy to work with any potential contributors on this to help them add this support. Please raise an issue first for any large feature change to ensure it is a good fit for this project before spending time on this work.
+Pull requests from those that know the missing syntax or dialects are especially
+welcomed and are the question way for you to get support added. We are happy
+to work with any potential contributors on this to help them add this support.
+Please raise an issue first for any large feature change to ensure it is a good
+fit for this project before spending time on this work.
 
 ## Templates Supported
 
-SQL itself does not lend itself well to [modularity](https://docs.getdbt.com/docs/viewpoint#section-modularity), so to introduce some flexibility and reusability it is often [templated](https://en.wikipedia.org/wiki/Template_processor) as discussed more in [our modularity documentation](https://docs.sqlfluff.com/en/stable/realworld.html#modularity).
+SQL itself does not lend itself well to [modularity](https://docs.getdbt.com/docs/viewpoint#section-modularity),
+so to introduce some flexibility and reusability it is often [templated](https://en.wikipedia.org/wiki/Template_processor)
+as discussed more in [our modularity documentation](https://docs.sqlfluff.com/en/stable/perma/modularity.html).
 
 **SQLFluff** supports the following templates:
+
 - [Jinja](https://jinja.palletsprojects.com/) (aka Jinja2)
-- [dbt](https://www.getdbt.com/)
+- SQL placeholders (e.g. SQLAlchemy parameters)
+- [Python format strings](https://docs.python.org/3/library/string.html#format-string-syntax)
+- [dbt](https://www.getdbt.com/) (requires plugin)
 
 Again, please raise issues if you wish to support more templating languages/syntaxes.
 
+## VS Code Extension
+
+We also have a VS Code extension:
+
+- [Github Repository](https://github.com/sqlfluff/vscode-sqlfluff)
+- [Extension in VS Code marketplace](https://marketplace.visualstudio.com/items?itemName=dorzey.vscode-sqlfluff)
+
 # Getting Started
 
 To get started, install the package and run `sqlfluff lint` or `sqlfluff fix`.
@@ -54,39 +105,77 @@ To get started, install the package and run `sqlfluff lint` or `sqlfluff fix`.
 ```shell
 $ pip install sqlfluff
 $ echo "  SELECT a  +  b FROM tbl;  " > test.sql
-$ sqlfluff lint test.sql
+$ sqlfluff lint test.sql --dialect ansi
 == [test.sql] FAIL
-L:   1 | P:   1 | L003 | Single indentation uses a number of spaces not a multiple of 4
-L:   1 | P:  14 | L006 | Operators should be surrounded by a single space unless at the start/end of a line
-L:   1 | P:  27 | L001 | Unnecessary trailing whitespace
+L:   1 | P:   1 | LT01 | Expected only single space before 'SELECT' keyword.
+                       | Found '  '. [layout.spacing]
+L:   1 | P:   1 | LT02 | First line should not be indented.
+                       | [layout.indent]
+L:   1 | P:   1 | LT13 | Files must not begin with newlines or whitespace.
+                       | [layout.start_of_file]
+L:   1 | P:  11 | LT01 | Expected only single space before binary operator '+'.
+                       | Found '  '. [layout.spacing]
+L:   1 | P:  14 | LT01 | Expected only single space before naked identifier.
+                       | Found '  '. [layout.spacing]
+L:   1 | P:  27 | LT01 | Unnecessary trailing whitespace at end of file.
+                       | [layout.spacing]
+L:   1 | P:  27 | LT12 | Files must end with a single trailing newline.
+                       | [layout.end_of_file]
+All Finished 📜 🎉!
 ```
 
-Alternatively, you can use the [**Official SQLFluff Docker Image**](https://hub.docker.com/r/sqlfluff/sqlfluff) or have a play using [**SQLFluff online**](https://online.sqlfluff.com/).
+Alternatively, you can use the [**Official SQLFluff Docker Image**](https://hub.docker.com/r/sqlfluff/sqlfluff)
+or have a play using [**SQLFluff online**](https://online.sqlfluff.com/).
 
-For full [CLI usage](https://docs.sqlfluff.com/en/stable/cli.html) and [rules reference](https://docs.sqlfluff.com/en/stable/rules.html), see [the SQLFluff docs](https://docs.sqlfluff.com/en/stable/).
+For full [CLI usage](https://docs.sqlfluff.com/en/stable/perma/cli.html) and
+[rules reference](https://docs.sqlfluff.com/en/stable/perma/rules.html), see
+[the SQLFluff docs](https://docs.sqlfluff.com/en/stable/).
 
 # Documentation
 
-For full documentation visit [docs.sqlfluff.com](https://docs.sqlfluff.com/en/stable/). This documentation is generated from this repository so please raise [issues](https://github.com/sqlfluff/sqlfluff/issues) or pull requests for any additions, corrections, or clarifications.
+For full documentation visit [docs.sqlfluff.com](https://docs.sqlfluff.com/en/stable/).
+This documentation is generated from this repository so please raise
+[issues](https://github.com/sqlfluff/sqlfluff/issues) or pull requests
+for any additions, corrections, or clarifications.
 
 # Releases
 
-**SQLFluff** is in beta phase - expect the tool to change significantly with potentially non-backwards compatible API and configuration changes in future releases. If you would like to join in please consider [contributing](CONTRIBUTING.md).
+**SQLFluff** adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html),
+so breaking changes should be restricted to major versions releases. Some
+elements (such as the python API) are in a less stable state and may see more
+significant changes more often. For details on breaking changes and how
+to migrate between versions, see our
+[release notes](https://docs.sqlfluff.com/en/latest/perma/releasenotes.html). See the
+[changelog](CHANGELOG.md) for more details. If you would like to join in, please
+consider [contributing](CONTRIBUTING.md).
 
-New releases are made monthly. For more information, visit [Releases](https://github.com/sqlfluff/sqlfluff/releases).
+New releases are made monthly. For more information, visit
+[Releases](https://github.com/sqlfluff/sqlfluff/releases).
 
 # SQLFluff on Slack
 
-We have a fast-growing community [on Slack](https://join.slack.com/t/sqlfluff/shared_invite/zt-o1f4x0e8-pZzarAIlQmKj_6ZwD16w0g), come and join us!
+We have a fast-growing community
+[on Slack](https://join.slack.com/t/sqlfluff/shared_invite/zt-2qtu36kdt-OS4iONPbQ3aCz2DIbYJdWg),
+come and join us!
 
 # SQLFluff on Twitter
 
-Follow us [on Twitter @SQLFluff](https://twitter.com/SQLFluff) for announcements and other related posts.
+Follow us [on Twitter @SQLFluff](https://twitter.com/SQLFluff) for announcements
+and other related posts.
 
 # Contributing
 
-We are grateful to all our [contributors](https://github.com/sqlfluff/sqlfluff/graphs/contributors). There is a lot to do in this project, and we are just getting started.
+We are grateful to all our [contributors](https://github.com/sqlfluff/sqlfluff/graphs/contributors).
+There is a lot to do in this project, and we are just getting started.
 
-If you want to understand more about the architecture of **SQLFluff**, you can find [more here](https://docs.sqlfluff.com/en/latest/architecture.html).
+If you want to understand more about the architecture of **SQLFluff**, you can
+find [more here](https://docs.sqlfluff.com/en/latest/perma/architecture.html).
 
-If you would like to contribute, check out the [open issues on GitHub](https://github.com/sqlfluff/sqlfluff/issues). You can also see the guide to [contributing](CONTRIBUTING.md).
+If you would like to contribute, check out the
+[open issues on GitHub](https://github.com/sqlfluff/sqlfluff/issues). You can also see
+the guide to [contributing](CONTRIBUTING.md).
+
+# Sponsors
+
+<img src="images/datacoves.png" alt="Datacoves" width="150"/><br>
+The turnkey analytics stack, find out more at [Datacoves.com](https://datacoves.com/).

benchmarks/bench_001_package.sql

@@ -1,92 +0,0 @@
--- Package model
-
-{{
-    config(
-        materialized = "table",
-    )
-}}
-
-with
-	wer as (
-		SELECT
-			sp.sergvsdrvs,
-			sp.sdtrsbnt,
-			trim(LTRIM(sp.sresgdr, 'asecesf')) as srebstrgserg,
-			sp.bdfgsrg,
-			sp.sfnsfdgnfd,
-            sp.vsdfbvsdfv,
-			sp.sdbsdr,
-			sp.srgdsrbsfcgb,
-            s.sdrgsdrbsd,
-            s.sdrgngf,
-            s.cvyjhcth,
-			tspc.fgyjmgbhmv,
-			tspc.cgh,
-			tspc.ghftdnftbcfhcgnc,
-			tspc.ynvgnmbjmvb,
-			s.vgbhm,
-			fhdtdtnyjftgjyumh,
-			sp.fgufghjfghjf
-		FROM {{ ref('wetrghdxfh') }} as sp
-            inner join {{ ref('strghtdfh') }} s using(sergvsdrvs)
-		left join {{ ref('sdrtghsdtrh') }} as tspc
-			on (sp.sdtrsbnt = tspc.sdtrsbnt)
-	),
-	qwe as (
-	select
-		servsdrfvsdzrv,
-		min(dsfgsergsrdg) as sftgbnsbgvszd
-	from {{ ref('sdfgsre') }}
-	group by servsdrfvsdzrv
-	),
-	ert as (
-		SELECT
-			p.sdfsdgsdfg,
-			MAX(IFF(ce.ts is not null, 1, 0)) = 1 as has_events,
-			min(ce.ts) as first_event,
-			max(ce.ts) as last_event
-		FROM sdfsgdfg p
-		LEFT JOIN {{ ref('dsrgsdrg') }} ce
-			on (p.dfgsd = trim(ce.lpn)
-				and ce.ts > p.sdfg - interval '30 days'
-				and ce.ts < p.sdfg + interval '60 days'
-                and ce.ts < CURRENT_DATE + interval '78 hours')
-		GROUP BY p.sdfsdgsdfg
-	),
-        dsfg as (
-            SELECT
-                p.rfgsrdsrd,
-                MAX(IFF(t.success = 0, 1, 0)) = 1 as sergsdrg
-            FROM wer p 
-            LEFT JOIN {{ ref('ncvbncbvnvcn') }} t
-                ON (p.dfg = t.dfg AND t.ertwretwetr = 'purchase')
-            GROUP BY p.rfgsrdsrd
-        )
-select
-	p.sdfgsdg,
-	p.wertwert,
-	p.nfghncvn,
-	p.fgsgdfg,
-	p.dfgsncn,
-    p.sdfhgdg,
-	p.ghdstrh,
-	p.dgnsfnstrh,
-	p.srthsdhfgh,
-	p.fgdfhgdfgn,
-	p.dfgnhdndtf,
-	p.dfthstghsreg,
-	qwe.sdfbsrb,
-	qwe.sdfbsfdb,
-	qwe.dfdfgdr,
-    billing_events.ahreagre,
-    p.fsdgseag,
-	p.fb,
-	p.fsgfdg,
-	od.xcbrdbrbsdrbsg,
-	p.sdfgsrbsrebs,
-	p.sdfgsdfbsdrg,
-	p.sdgsdrgrgrdgs
-from packages p
-inner join qwe using(sdrgsrdg)
-inner join ert using(sdfasef)
-INNER JOIN dsfg od ON p.shipment_id = od.shipment_id

benchmarks/bench_002/.sqlfluff

@@ -1,34 +0,0 @@
-[sqlfluff]
-verbose = 0
-nocolor = False
-dialect = bigquery
-templater = jinja
-rules = None
-exclude_rules = None
-recurse = 0
-
-[sqlfluff:templater:jinja:macros]
-
-# Some rules can be configured directly from the config common to other rules.
-[sqlfluff:rules]
-tab_space_size = 4
-
-# Some rules have their own specific config.
-[sqlfluff:rules:L010]
-capitalisation_policy = consistent
-
-[sqlfluff:templater:jinja:context]
-dataset=dataset
-gcp_project=gcp_project
-benchmark_user_map_project=project
-benchmark_user_map_dataset=summary
-benchmark_user_map_table=benchmark_user_map
-benchmark_summaries_project=project
-benchmark_summaries_dataset=summary
-benchmark_summaries_table=benchmark_summaries
-campaign_performance_project=project
-campaign_performance_dataset=summary
-campaign_performance_table=campaign_performance
-user_average_project=project
-user_average_dataset=summary
-user_average_table=average_user_performance

benchmarks/bench_002/bench_002_pearson.sql

@@ -1,246 +0,0 @@
--- This query generated by script/generate_corr_queries.py and should probably not be
---  modified manually. Instead, make changes to that script and rerun it.
-
-WITH
-  raw_effect_sizes AS (
-  SELECT
-    COUNT(1) AS campaign_count,
-    state_user_v_peer_open
- ,business_type
-    
-    -- The following is the slope of the regression line. Note that CORR (which is the Pearson's correlation
-    --  coefficient is symmetric in its arguments, but since STDDEV_POP(open_rate_su) appears in the
-    --  numerator this is the slope of the regression line considering STDDEV_POP(open_rate_su) to be
-    --  the "y variable" and uses_small_subject_line to be the "x variable" in terms of the regression line.
-    ,SAFE_DIVIDE(SAFE_MULTIPLY(CORR(open_rate_su, uses_small_subject_line), STDDEV_POP(open_rate_su)), STDDEV_POP(uses_small_subject_line)) AS open_uses_small_subject_line
-    
-    -- The following is the slope of the regression line. Note that CORR (which is the Pearson's correlation
-    --  coefficient is symmetric in its arguments, but since STDDEV_POP(open_rate_su) appears in the
-    --  numerator this is the slope of the regression line considering STDDEV_POP(open_rate_su) to be
-    --  the "y variable" and uses_personal_subject to be the "x variable" in terms of the regression line.
-    ,SAFE_DIVIDE(SAFE_MULTIPLY(CORR(open_rate_su, uses_personal_subject), STDDEV_POP(open_rate_su)), STDDEV_POP(uses_personal_subject)) AS open_uses_personal_subject
-    
-    -- The following is the slope of the regression line. Note that CORR (which is the Pearson's correlation
-    --  coefficient is symmetric in its arguments, but since STDDEV_POP(open_rate_su) appears in the
-    --  numerator this is the slope of the regression line considering STDDEV_POP(open_rate_su) to be
-    --  the "y variable" and uses_timewarp to be the "x variable" in terms of the regression line.
-    ,SAFE_DIVIDE(SAFE_MULTIPLY(CORR(open_rate_su, uses_timewarp), STDDEV_POP(open_rate_su)), STDDEV_POP(uses_timewarp)) AS open_uses_timewarp
-    
-    -- The following is the slope of the regression line. Note that CORR (which is the Pearson's correlation
-    --  coefficient is symmetric in its arguments, but since STDDEV_POP(open_rate_su) appears in the
-    --  numerator this is the slope of the regression line considering STDDEV_POP(open_rate_su) to be
-    --  the "y variable" and uses_small_preview to be the "x variable" in terms of the regression line.
-    ,SAFE_DIVIDE(SAFE_MULTIPLY(CORR(open_rate_su, uses_small_preview), STDDEV_POP(open_rate_su)), STDDEV_POP(uses_small_preview)) AS open_uses_small_preview
-    
-    -- The following is the slope of the regression line. Note that CORR (which is the Pearson's correlation
-    --  coefficient is symmetric in its arguments, but since STDDEV_POP(open_rate_su) appears in the
-    --  numerator this is the slope of the regression line considering STDDEV_POP(open_rate_su) to be
-    --  the "y variable" and uses_personal_to to be the "x variable" in terms of the regression line.
-    ,SAFE_DIVIDE(SAFE_MULTIPLY(CORR(open_rate_su, uses_personal_to), STDDEV_POP(open_rate_su)), STDDEV_POP(uses_personal_to)) AS open_uses_personal_to
-    
-    -- The following is the slope of the regression line. Note that CORR (which is the Pearson's correlation
-    --  coefficient is symmetric in its arguments, but since STDDEV_POP(open_rate_su) appears in the
-    --  numerator this is the slope of the regression line considering STDDEV_POP(open_rate_su) to be
-    --  the "y variable" and uses_ab_test_subject to be the "x variable" in terms of the regression line.
-    ,SAFE_DIVIDE(SAFE_MULTIPLY(CORR(open_rate_su, uses_ab_test_subject), STDDEV_POP(open_rate_su)), STDDEV_POP(uses_ab_test_subject)) AS open_uses_ab_test_subject
-    
-    -- The following is the slope of the regression line. Note that CORR (which is the Pearson's correlation
-    --  coefficient is symmetric in its arguments, but since STDDEV_POP(open_rate_su) appears in the
-    --  numerator this is the slope of the regression line considering STDDEV_POP(open_rate_su) to be
-    --  the "y variable" and uses_ab_test_content to be the "x variable" in terms of the regression line.
-    ,SAFE_DIVIDE(SAFE_MULTIPLY(CORR(open_rate_su, uses_ab_test_content), STDDEV_POP(open_rate_su)), STDDEV_POP(uses_ab_test_content)) AS open_uses_ab_test_content
-    
-    -- The following is the slope of the regression line. Note that CORR (which is the Pearson's correlation
-    --  coefficient is symmetric in its arguments, but since STDDEV_POP(open_rate_su) appears in the
-    --  numerator this is the slope of the regression line considering STDDEV_POP(open_rate_su) to be
-    --  the "y variable" and uses_preview_text to be the "x variable" in terms of the regression line.
-    ,SAFE_DIVIDE(SAFE_MULTIPLY(CORR(open_rate_su, uses_preview_text), STDDEV_POP(open_rate_su)), STDDEV_POP(uses_preview_text)) AS open_uses_preview_text
-    
-    -- The following is the slope of the regression line. Note that CORR (which is the Pearson's correlation
-    --  coefficient is symmetric in its arguments, but since STDDEV_POP(open_rate_su) appears in the
-    --  numerator this is the slope of the regression line considering STDDEV_POP(open_rate_su) to be
-    --  the "y variable" and uses_sto to be the "x variable" in terms of the regression line.
-    ,SAFE_DIVIDE(SAFE_MULTIPLY(CORR(open_rate_su, uses_sto), STDDEV_POP(open_rate_su)), STDDEV_POP(uses_sto)) AS open_uses_sto
-    
-    -- The following is the slope of the regression line. Note that CORR (which is the Pearson's correlation
-    --  coefficient is symmetric in its arguments, but since STDDEV_POP(open_rate_su) appears in the
-    --  numerator this is the slope of the regression line considering STDDEV_POP(open_rate_su) to be
-    --  the "y variable" and uses_freemail_from to be the "x variable" in terms of the regression line.
-    ,SAFE_DIVIDE(SAFE_MULTIPLY(CORR(open_rate_su, uses_freemail_from), STDDEV_POP(open_rate_su)), STDDEV_POP(uses_freemail_from)) AS open_uses_freemail_from
-    
-    -- The following is the slope of the regression line. Note that CORR (which is the Pearson's correlation
-    --  coefficient is symmetric in its arguments, but since STDDEV_POP(open_rate_su) appears in the
-    --  numerator this is the slope of the regression line considering STDDEV_POP(open_rate_su) to be
-    --  the "y variable" and uses_resend_non_openers to be the "x variable" in terms of the regression line.
-    ,SAFE_DIVIDE(SAFE_MULTIPLY(CORR(open_rate_su, uses_resend_non_openers), STDDEV_POP(open_rate_su)), STDDEV_POP(uses_resend_non_openers)) AS open_uses_resend_non_openers
-    
-    -- The following is the slope of the regression line. Note that CORR (which is the Pearson's correlation
-    --  coefficient is symmetric in its arguments, but since STDDEV_POP(open_rate_su) appears in the
-    --  numerator this is the slope of the regression line considering STDDEV_POP(open_rate_su) to be
-    --  the "y variable" and uses_promo_code to be the "x variable" in terms of the regression line.
-    ,SAFE_DIVIDE(SAFE_MULTIPLY(CORR(open_rate_su, uses_promo_code), STDDEV_POP(open_rate_su)), STDDEV_POP(uses_promo_code)) AS open_uses_promo_code
-    
-    -- The following is the slope of the regression line. Note that CORR (which is the Pearson's correlation
-    --  coefficient is symmetric in its arguments, but since STDDEV_POP(open_rate_su) appears in the
-    --  numerator this is the slope of the regression line considering STDDEV_POP(open_rate_su) to be
-    --  the "y variable" and uses_prex to be the "x variable" in terms of the regression line.
-    ,SAFE_DIVIDE(SAFE_MULTIPLY(CORR(open_rate_su, uses_prex), STDDEV_POP(open_rate_su)), STDDEV_POP(uses_prex)) AS open_uses_prex
-    
-    -- The following is the slope of the regression line. Note that CORR (which is the Pearson's correlation
-    --  coefficient is symmetric in its arguments, but since STDDEV_POP(open_rate_su) appears in the
-    --  numerator this is the slope of the regression line considering STDDEV_POP(open_rate_su) to be
-    --  the "y variable" and uses_ab_test_from to be the "x variable" in terms of the regression line.
-    ,SAFE_DIVIDE(SAFE_MULTIPLY(CORR(open_rate_su, uses_ab_test_from), STDDEV_POP(open_rate_su)), STDDEV_POP(uses_ab_test_from)) AS open_uses_ab_test_from
-    
-  FROM
-    `{{gcp_project}}.{{dataset}}.global_actions_states`
-  GROUP BY
-    state_user_v_peer_open
- ,business_type),
-
-  imputed_effect_sizes AS (
-  SELECT
-    campaign_count,
-    state_user_v_peer_open
- ,business_type
-    
-    -- We now impute the value of the effect size to 0 if it was NaN or NULL. This is to
-    --  take into account states where all campaigns either did or did not perform an 
-    --  action. In these cases, we assume that campaign outcome is uncorrelated with 
-    --  the action because we do not have evidence otherwise.
-    ,COALESCE(IF(IS_NAN(open_uses_small_subject_line), 0, open_uses_small_subject_line), 0) AS open_uses_small_subject_line
-    
-    -- We now impute the value of the effect size to 0 if it was NaN or NULL. This is to
-    --  take into account states where all campaigns either did or did not perform an 
-    --  action. In these cases, we assume that campaign outcome is uncorrelated with 
-    --  the action because we do not have evidence otherwise.
-    ,COALESCE(IF(IS_NAN(open_uses_personal_subject), 0, open_uses_personal_subject), 0) AS open_uses_personal_subject
-    
-    -- We now impute the value of the effect size to 0 if it was NaN or NULL. This is to
-    --  take into account states where all campaigns either did or did not perform an 
-    --  action. In these cases, we assume that campaign outcome is uncorrelated with 
-    --  the action because we do not have evidence otherwise.
-    ,COALESCE(IF(IS_NAN(open_uses_timewarp), 0, open_uses_timewarp), 0) AS open_uses_timewarp
-    
-    -- We now impute the value of the effect size to 0 if it was NaN or NULL. This is to
-    --  take into account states where all campaigns either did or did not perform an 
-    --  action. In these cases, we assume that campaign outcome is uncorrelated with 
-    --  the action because we do not have evidence otherwise.
-    ,COALESCE(IF(IS_NAN(open_uses_small_preview), 0, open_uses_small_preview), 0) AS open_uses_small_preview
-    
-    -- We now impute the value of the effect size to 0 if it was NaN or NULL. This is to
-    --  take into account states where all campaigns either did or did not perform an 
-    --  action. In these cases, we assume that campaign outcome is uncorrelated with 
-    --  the action because we do not have evidence otherwise.
-    ,COALESCE(IF(IS_NAN(open_uses_personal_to), 0, open_uses_personal_to), 0) AS open_uses_personal_to
-    
-    -- We now impute the value of the effect size to 0 if it was NaN or NULL. This is to
-    --  take into account states where all campaigns either did or did not perform an 
-    --  action. In these cases, we assume that campaign outcome is uncorrelated with 
-    --  the action because we do not have evidence otherwise.
-    ,COALESCE(IF(IS_NAN(open_uses_ab_test_subject), 0, open_uses_ab_test_subject), 0) AS open_uses_ab_test_subject
-    
-    -- We now impute the value of the effect size to 0 if it was NaN or NULL. This is to
-    --  take into account states where all campaigns either did or did not perform an 
-    --  action. In these cases, we assume that campaign outcome is uncorrelated with 
-    --  the action because we do not have evidence otherwise.
-    ,COALESCE(IF(IS_NAN(open_uses_ab_test_content), 0, open_uses_ab_test_content), 0) AS open_uses_ab_test_content
-    
-    -- We now impute the value of the effect size to 0 if it was NaN or NULL. This is to
-    --  take into account states where all campaigns either did or did not perform an 
-    --  action. In these cases, we assume that campaign outcome is uncorrelated with 
-    --  the action because we do not have evidence otherwise.
-    ,COALESCE(IF(IS_NAN(open_uses_preview_text), 0, open_uses_preview_text), 0) AS open_uses_preview_text
-    
-    -- We now impute the value of the effect size to 0 if it was NaN or NULL. This is to
-    --  take into account states where all campaigns either did or did not perform an 
-    --  action. In these cases, we assume that campaign outcome is uncorrelated with 
-    --  the action because we do not have evidence otherwise.
-    ,COALESCE(IF(IS_NAN(open_uses_sto), 0, open_uses_sto), 0) AS open_uses_sto
-    
-    -- We now impute the value of the effect size to 0 if it was NaN or NULL. This is to
-    --  take into account states where all campaigns either did or did not perform an 
-    --  action. In these cases, we assume that campaign outcome is uncorrelated with 
-    --  the action because we do not have evidence otherwise.
-    ,COALESCE(IF(IS_NAN(open_uses_freemail_from), 0, open_uses_freemail_from), 0) AS open_uses_freemail_from
-    
-    -- We now impute the value of the effect size to 0 if it was NaN or NULL. This is to
-    --  take into account states where all campaigns either did or did not perform an 
-    --  action. In these cases, we assume that campaign outcome is uncorrelated with 
-    --  the action because we do not have evidence otherwise.
-    ,COALESCE(IF(IS_NAN(open_uses_resend_non_openers), 0, open_uses_resend_non_openers), 0) AS open_uses_resend_non_openers
-    
-    -- We now impute the value of the effect size to 0 if it was NaN or NULL. This is to
-    --  take into account states where all campaigns either did or did not perform an 
-    --  action. In these cases, we assume that campaign outcome is uncorrelated with 
-    --  the action because we do not have evidence otherwise.
-    ,COALESCE(IF(IS_NAN(open_uses_promo_code), 0, open_uses_promo_code), 0) AS open_uses_promo_code
-    
-    -- We now impute the value of the effect size to 0 if it was NaN or NULL. This is to
-    --  take into account states where all campaigns either did or did not perform an 
-    --  action. In these cases, we assume that campaign outcome is uncorrelated with 
-    --  the action because we do not have evidence otherwise.
-    ,COALESCE(IF(IS_NAN(open_uses_prex), 0, open_uses_prex), 0) AS open_uses_prex
-    
-    -- We now impute the value of the effect size to 0 if it was NaN or NULL. This is to
-    --  take into account states where all campaigns either did or did not perform an 
-    --  action. In these cases, we assume that campaign outcome is uncorrelated with 
-    --  the action because we do not have evidence otherwise.
-    ,COALESCE(IF(IS_NAN(open_uses_ab_test_from), 0, open_uses_ab_test_from), 0) AS open_uses_ab_test_from
-    
-  FROM
-    raw_effect_sizes
-  ),
-
-  action_states AS (
-  SELECT 
-    has_used_small_subject_line
- ,has_used_personal_subject
- ,has_used_timewarp
- ,has_used_small_preview
- ,has_used_personal_to
- ,has_used_ab_test_subject
- ,has_used_ab_test_content
- ,has_used_preview_text
- ,has_used_sto
- ,has_used_freemail_from
- ,has_used_resend_non_openers
- ,has_used_promo_code
- ,has_used_prex
- ,has_used_ab_test_from
-  FROM `{{gcp_project}}.{{dataset}}.global_state_space`
-  GROUP BY has_used_small_subject_line
- ,has_used_personal_subject
- ,has_used_timewarp
- ,has_used_small_preview
- ,has_used_personal_to
- ,has_used_ab_test_subject
- ,has_used_ab_test_content
- ,has_used_preview_text
- ,has_used_sto
- ,has_used_freemail_from
- ,has_used_resend_non_openers
- ,has_used_promo_code
- ,has_used_prex
- ,has_used_ab_test_from)
-
-SELECT
-  imputed_effect_sizes.*,
-  has_used_small_subject_line
- ,has_used_personal_subject
- ,has_used_timewarp
- ,has_used_small_preview
- ,has_used_personal_to
- ,has_used_ab_test_subject
- ,has_used_ab_test_content
- ,has_used_preview_text
- ,has_used_sto
- ,has_used_freemail_from
- ,has_used_resend_non_openers
- ,has_used_promo_code
- ,has_used_prex
- ,has_used_ab_test_from
-FROM
-  imputed_effect_sizes
-CROSS JOIN action_states
-ORDER BY campaign_count DESC

benchmarks/bench_002/bench_002_pearson_fix.sql

@@ -1,370 +0,0 @@
--- This query generated by script/generate_corr_queries.py and should probably not be
---  modified manually. Instead, make changes to that script and rerun it.
-
-WITH
-raw_effect_sizes AS (
-    SELECT
-        state_user_v_peer_open,
-        business_type,
-        COUNT(*) AS campaign_count,
-
-        -- The following is the slope of the regression line. Note that CORR (which is the Pearson's correlation
-        --  coefficient is symmetric in its arguments, but since STDDEV_POP(open_rate_su) appears in the
-        --  numerator this is the slope of the regression line considering STDDEV_POP(open_rate_su) to be
-        --  the "y variable" and uses_small_subject_line to be the "x variable" in terms of the regression line.
-        SAFE_DIVIDE(
-             SAFE_MULTIPLY(
-                CORR(open_rate_su, uses_small_subject_line),
-                STDDEV_POP(open_rate_su)
-            ),
-             STDDEV_POP(uses_small_subject_line)
-        ) AS open_uses_small_subject_line,
-
-        -- The following is the slope of the regression line. Note that CORR (which is the Pearson's correlation
-        --  coefficient is symmetric in its arguments, but since STDDEV_POP(open_rate_su) appears in the
-        --  numerator this is the slope of the regression line considering STDDEV_POP(open_rate_su) to be
-        --  the "y variable" and uses_personal_subject to be the "x variable" in terms of the regression line.
-        SAFE_DIVIDE(
-             SAFE_MULTIPLY(
-                CORR(open_rate_su, uses_personal_subject),
-                STDDEV_POP(open_rate_su)
-            ),
-             STDDEV_POP(uses_personal_subject)
-        ) AS open_uses_personal_subject,
-
-        -- The following is the slope of the regression line. Note that CORR (which is the Pearson's correlation
-        --  coefficient is symmetric in its arguments, but since STDDEV_POP(open_rate_su) appears in the
-        --  numerator this is the slope of the regression line considering STDDEV_POP(open_rate_su) to be
-        --  the "y variable" and uses_timewarp to be the "x variable" in terms of the regression line.
-        SAFE_DIVIDE(
-             SAFE_MULTIPLY(
-                CORR(open_rate_su, uses_timewarp), STDDEV_POP(open_rate_su)
-            ),
-             STDDEV_POP(uses_timewarp)
-        ) AS open_uses_timewarp,
-
-        -- The following is the slope of the regression line. Note that CORR (which is the Pearson's correlation
-        --  coefficient is symmetric in its arguments, but since STDDEV_POP(open_rate_su) appears in the
-        --  numerator this is the slope of the regression line considering STDDEV_POP(open_rate_su) to be
-        --  the "y variable" and uses_small_preview to be the "x variable" in terms of the regression line.
-        SAFE_DIVIDE(
-             SAFE_MULTIPLY(
-                CORR(open_rate_su, uses_small_preview), STDDEV_POP(open_rate_su)
-            ),
-             STDDEV_POP(uses_small_preview)
-        ) AS open_uses_small_preview,
-
-        -- The following is the slope of the regression line. Note that CORR (which is the Pearson's correlation
-        --  coefficient is symmetric in its arguments, but since STDDEV_POP(open_rate_su) appears in the
-        --  numerator this is the slope of the regression line considering STDDEV_POP(open_rate_su) to be
-        --  the "y variable" and uses_personal_to to be the "x variable" in terms of the regression line.
-        SAFE_DIVIDE(
-             SAFE_MULTIPLY(
-                CORR(open_rate_su, uses_personal_to), STDDEV_POP(open_rate_su)
-            ),
-             STDDEV_POP(uses_personal_to)
-        ) AS open_uses_personal_to,
-
-        -- The following is the slope of the regression line. Note that CORR (which is the Pearson's correlation
-        --  coefficient is symmetric in its arguments, but since STDDEV_POP(open_rate_su) appears in the
-        --  numerator this is the slope of the regression line considering STDDEV_POP(open_rate_su) to be
-        --  the "y variable" and uses_ab_test_subject to be the "x variable" in terms of the regression line.
-        SAFE_DIVIDE(
-             SAFE_MULTIPLY(
-                CORR(open_rate_su, uses_ab_test_subject),
-                STDDEV_POP(open_rate_su)
-            ),
-             STDDEV_POP(uses_ab_test_subject)
-        ) AS open_uses_ab_test_subject,
-
-        -- The following is the slope of the regression line. Note that CORR (which is the Pearson's correlation
-        --  coefficient is symmetric in its arguments, but since STDDEV_POP(open_rate_su) appears in the
-        --  numerator this is the slope of the regression line considering STDDEV_POP(open_rate_su) to be
-        --  the "y variable" and uses_ab_test_content to be the "x variable" in terms of the regression line.
-        SAFE_DIVIDE(
-             SAFE_MULTIPLY(
-                CORR(open_rate_su, uses_ab_test_content),
-                STDDEV_POP(open_rate_su)
-            ),
-             STDDEV_POP(uses_ab_test_content)
-        ) AS open_uses_ab_test_content,
-
-        -- The following is the slope of the regression line. Note that CORR (which is the Pearson's correlation
-        --  coefficient is symmetric in its arguments, but since STDDEV_POP(open_rate_su) appears in the
-        --  numerator this is the slope of the regression line considering STDDEV_POP(open_rate_su) to be
-        --  the "y variable" and uses_preview_text to be the "x variable" in terms of the regression line.
-        SAFE_DIVIDE(
-             SAFE_MULTIPLY(
-                CORR(open_rate_su, uses_preview_text), STDDEV_POP(open_rate_su)
-            ),
-             STDDEV_POP(uses_preview_text)
-        ) AS open_uses_preview_text,
-
-        -- The following is the slope of the regression line. Note that CORR (which is the Pearson's correlation
-        --  coefficient is symmetric in its arguments, but since STDDEV_POP(open_rate_su) appears in the
-        --  numerator this is the slope of the regression line considering STDDEV_POP(open_rate_su) to be
-        --  the "y variable" and uses_sto to be the "x variable" in terms of the regression line.
-        SAFE_DIVIDE(
-             SAFE_MULTIPLY(
-                CORR(open_rate_su, uses_sto), STDDEV_POP(open_rate_su)
-            ),
-             STDDEV_POP(uses_sto)
-        ) AS open_uses_sto,
-
-        -- The following is the slope of the regression line. Note that CORR (which is the Pearson's correlation
-        --  coefficient is symmetric in its arguments, but since STDDEV_POP(open_rate_su) appears in the
-        --  numerator this is the slope of the regression line considering STDDEV_POP(open_rate_su) to be
-        --  the "y variable" and uses_freemail_from to be the "x variable" in terms of the regression line.
-        SAFE_DIVIDE(
-             SAFE_MULTIPLY(
-                CORR(open_rate_su, uses_freemail_from), STDDEV_POP(open_rate_su)
-            ),
-             STDDEV_POP(uses_freemail_from)
-        ) AS open_uses_freemail_from,
-
-        -- The following is the slope of the regression line. Note that CORR (which is the Pearson's correlation
-        --  coefficient is symmetric in its arguments, but since STDDEV_POP(open_rate_su) appears in the
-        --  numerator this is the slope of the regression line considering STDDEV_POP(open_rate_su) to be
-        --  the "y variable" and uses_resend_non_openers to be the "x variable" in terms of the regression line.
-        SAFE_DIVIDE(
-             SAFE_MULTIPLY(
-                CORR(open_rate_su, uses_resend_non_openers),
-                STDDEV_POP(open_rate_su)
-            ),
-             STDDEV_POP(uses_resend_non_openers)
-        ) AS open_uses_resend_non_openers,
-
-        -- The following is the slope of the regression line. Note that CORR (which is the Pearson's correlation
-        --  coefficient is symmetric in its arguments, but since STDDEV_POP(open_rate_su) appears in the
-        --  numerator this is the slope of the regression line considering STDDEV_POP(open_rate_su) to be
-        --  the "y variable" and uses_promo_code to be the "x variable" in terms of the regression line.
-        SAFE_DIVIDE(
-             SAFE_MULTIPLY(
-                CORR(open_rate_su, uses_promo_code), STDDEV_POP(open_rate_su)
-            ),
-             STDDEV_POP(uses_promo_code)
-        ) AS open_uses_promo_code,
-
-        -- The following is the slope of the regression line. Note that CORR (which is the Pearson's correlation
-        --  coefficient is symmetric in its arguments, but since STDDEV_POP(open_rate_su) appears in the
-        --  numerator this is the slope of the regression line considering STDDEV_POP(open_rate_su) to be
-        --  the "y variable" and uses_prex to be the "x variable" in terms of the regression line.
-        SAFE_DIVIDE(
-             SAFE_MULTIPLY(
-                CORR(open_rate_su, uses_prex), STDDEV_POP(open_rate_su)
-            ),
-             STDDEV_POP(uses_prex)
-        ) AS open_uses_prex,
-
-        -- The following is the slope of the regression line. Note that CORR (which is the Pearson's correlation
-        --  coefficient is symmetric in its arguments, but since STDDEV_POP(open_rate_su) appears in the
-        --  numerator this is the slope of the regression line considering STDDEV_POP(open_rate_su) to be
-        --  the "y variable" and uses_ab_test_from to be the "x variable" in terms of the regression line.
-        SAFE_DIVIDE(
-             SAFE_MULTIPLY(
-                CORR(open_rate_su, uses_ab_test_from), STDDEV_POP(open_rate_su)
-            ),
-             STDDEV_POP(uses_ab_test_from)
-        ) AS open_uses_ab_test_from
-
-    FROM
-        `{{gcp_project}}.{{dataset}}.global_actions_states`
-    GROUP BY
-        state_user_v_peer_open,
-        business_type),
-
-imputed_effect_sizes AS (
-    SELECT
-        campaign_count,
-        state_user_v_peer_open,
-        business_type,
-
-        -- We now impute the value of the effect size to 0 if it was NaN or NULL. This is to
-        --  take into account states where all campaigns either did or did not perform an 
-        --  action. In these cases, we assume that campaign outcome is uncorrelated with 
-        --  the action because we do not have evidence otherwise.
-        COALESCE(
-             IF(
-                IS_NAN(open_uses_small_subject_line),
-                0,
-                open_uses_small_subject_line
-            ),
-             0
-        ) AS open_uses_small_subject_line,
-
-        -- We now impute the value of the effect size to 0 if it was NaN or NULL. This is to
-        --  take into account states where all campaigns either did or did not perform an 
-        --  action. In these cases, we assume that campaign outcome is uncorrelated with 
-        --  the action because we do not have evidence otherwise.
-        COALESCE(
-             IF(
-                IS_NAN(open_uses_personal_subject),
-                0,
-                open_uses_personal_subject
-            ),
-            0
-        ) AS open_uses_personal_subject,
-
-        -- We now impute the value of the effect size to 0 if it was NaN or NULL. This is to
-        --  take into account states where all campaigns either did or did not perform an 
-        --  action. In these cases, we assume that campaign outcome is uncorrelated with 
-        --  the action because we do not have evidence otherwise.
-        COALESCE(
-             IF(IS_NAN(open_uses_timewarp), 0, open_uses_timewarp), 0
-        ) AS open_uses_timewarp,
-
-        -- We now impute the value of the effect size to 0 if it was NaN or NULL. This is to
-        --  take into account states where all campaigns either did or did not perform an 
-        --  action. In these cases, we assume that campaign outcome is uncorrelated with 
-        --  the action because we do not have evidence otherwise.
-        COALESCE(
-             IF(IS_NAN(open_uses_small_preview), 0, open_uses_small_preview), 0
-        ) AS open_uses_small_preview,
-
-        -- We now impute the value of the effect size to 0 if it was NaN or NULL. This is to
-        --  take into account states where all campaigns either did or did not perform an 
-        --  action. In these cases, we assume that campaign outcome is uncorrelated with 
-        --  the action because we do not have evidence otherwise.
-        COALESCE(
-             IF(IS_NAN(open_uses_personal_to), 0, open_uses_personal_to), 0
-        ) AS open_uses_personal_to,
-
-        -- We now impute the value of the effect size to 0 if it was NaN or NULL. This is to
-        --  take into account states where all campaigns either did or did not perform an 
-        --  action. In these cases, we assume that campaign outcome is uncorrelated with 
-        --  the action because we do not have evidence otherwise.
-        COALESCE(
-             IF(
-                IS_NAN(open_uses_ab_test_subject), 0, open_uses_ab_test_subject
-            ),
-            0
-        ) AS open_uses_ab_test_subject,
-
-        -- We now impute the value of the effect size to 0 if it was NaN or NULL. This is to
-        --  take into account states where all campaigns either did or did not perform an 
-        --  action. In these cases, we assume that campaign outcome is uncorrelated with 
-        --  the action because we do not have evidence otherwise.
-        COALESCE(
-             IF(
-                IS_NAN(open_uses_ab_test_content), 0, open_uses_ab_test_content
-            ),
-            0
-        ) AS open_uses_ab_test_content,
-
-        -- We now impute the value of the effect size to 0 if it was NaN or NULL. This is to
-        --  take into account states where all campaigns either did or did not perform an 
-        --  action. In these cases, we assume that campaign outcome is uncorrelated with 
-        --  the action because we do not have evidence otherwise.
-        COALESCE(
-             IF(IS_NAN(open_uses_preview_text), 0, open_uses_preview_text), 0
-        ) AS open_uses_preview_text,
-
-        -- We now impute the value of the effect size to 0 if it was NaN or NULL. This is to
-        --  take into account states where all campaigns either did or did not perform an 
-        --  action. In these cases, we assume that campaign outcome is uncorrelated with 
-        --  the action because we do not have evidence otherwise.
-        COALESCE(
-            IF(IS_NAN(open_uses_sto), 0, open_uses_sto), 0
-        ) AS open_uses_sto,
-
-        -- We now impute the value of the effect size to 0 if it was NaN or NULL. This is to
-        --  take into account states where all campaigns either did or did not perform an 
-        --  action. In these cases, we assume that campaign outcome is uncorrelated with 
-        --  the action because we do not have evidence otherwise.
-        COALESCE(
-             IF(IS_NAN(open_uses_freemail_from), 0, open_uses_freemail_from), 0
-        ) AS open_uses_freemail_from,
-
-        -- We now impute the value of the effect size to 0 if it was NaN or NULL. This is to
-        --  take into account states where all campaigns either did or did not perform an 
-        --  action. In these cases, we assume that campaign outcome is uncorrelated with 
-        --  the action because we do not have evidence otherwise.
-        COALESCE(
-             IF(
-                IS_NAN(open_uses_resend_non_openers),
-                0,
-                open_uses_resend_non_openers
-            ),
-             0
-        ) AS open_uses_resend_non_openers,
-
-        -- We now impute the value of the effect size to 0 if it was NaN or NULL. This is to
-        --  take into account states where all campaigns either did or did not perform an 
-        --  action. In these cases, we assume that campaign outcome is uncorrelated with 
-        --  the action because we do not have evidence otherwise.
-        COALESCE(
-             IF(IS_NAN(open_uses_promo_code), 0, open_uses_promo_code), 0
-        ) AS open_uses_promo_code,
-
-        -- We now impute the value of the effect size to 0 if it was NaN or NULL. This is to
-        --  take into account states where all campaigns either did or did not perform an 
-        --  action. In these cases, we assume that campaign outcome is uncorrelated with 
-        --  the action because we do not have evidence otherwise.
-        COALESCE(
-             IF(IS_NAN(open_uses_prex), 0, open_uses_prex), 0
-        ) AS open_uses_prex,
-
-        -- We now impute the value of the effect size to 0 if it was NaN or NULL. This is to
-        --  take into account states where all campaigns either did or did not perform an 
-        --  action. In these cases, we assume that campaign outcome is uncorrelated with 
-        --  the action because we do not have evidence otherwise.
-        COALESCE(
-             IF(IS_NAN(open_uses_ab_test_from), 0, open_uses_ab_test_from), 0
-        ) AS open_uses_ab_test_from
-
-    FROM
-        raw_effect_sizes
-),
-
-action_states AS (
-    SELECT
-        has_used_small_subject_line,
-        has_used_personal_subject,
-        has_used_timewarp,
-        has_used_small_preview,
-        has_used_personal_to,
-        has_used_ab_test_subject,
-        has_used_ab_test_content,
-        has_used_preview_text,
-        has_used_sto,
-        has_used_freemail_from,
-        has_used_resend_non_openers,
-        has_used_promo_code,
-        has_used_prex,
-        has_used_ab_test_from
-    FROM `{{gcp_project}}.{{dataset}}.global_state_space`
-    GROUP BY has_used_small_subject_line,
-        has_used_personal_subject,
-        has_used_timewarp,
-        has_used_small_preview,
-        has_used_personal_to,
-        has_used_ab_test_subject,
-        has_used_ab_test_content,
-        has_used_preview_text,
-        has_used_sto,
-        has_used_freemail_from,
-        has_used_resend_non_openers,
-        has_used_promo_code,
-        has_used_prex,
-        has_used_ab_test_from)
-
-SELECT
-    imputed_effect_sizes.*,
-    has_used_small_subject_line,
-    has_used_personal_subject,
-    has_used_timewarp,
-    has_used_small_preview,
-    has_used_personal_to,
-    has_used_ab_test_subject,
-    has_used_ab_test_content,
-    has_used_preview_text,
-    has_used_sto,
-    has_used_freemail_from,
-    has_used_resend_non_openers,
-    has_used_promo_code,
-    has_used_prex,
-    has_used_ab_test_from
-FROM
-    imputed_effect_sizes
-CROSS JOIN action_states
-ORDER BY campaign_count DESC

benchmarks/benchmarks.yml

@@ -1,8 +0,0 @@
-benchmarks:
-    - name: 004_L003_indentation_3
-      cmd: ['sqlfluff', 'parse', '--bench', 'test/fixtures/linter/autofix/ansi/004_indentation/before.sql']
-    - name: B_001_package
-      cmd: ['sqlfluff', 'parse', '--bench', 'benchmarks/bench_001_package.sql']
-    - name: B_002_pearson
-      cmd: ['sqlfluff', 'fix', '-f', '--bench',
-            '--fixed-suffix', '_fix', 'benchmarks/bench_002/bench_002_pearson.sql']

constraints/dbt140.txt

@@ -0,0 +1,2 @@
+dbt-core~=1.4.1
+dbt-postgres~=1.4.1

constraints/dbt150-winpy.txt

@@ -0,0 +1,2 @@
+dbt-core~=1.5.0
+dbt-postgres~=1.5.0

constraints/dbt150.txt

@@ -0,0 +1,2 @@
+dbt-core~=1.5.0
+dbt-postgres~=1.5.0

constraints/dbt160.txt

@@ -0,0 +1,2 @@
+dbt-core~=1.6.0
+dbt-postgres~=1.6.0

constraints/dbt170.txt

@@ -0,0 +1,2 @@
+dbt-core~=1.7.0
+dbt-postgres~=1.7.0

constraints/dbt180-winpy.txt

@@ -0,0 +1,2 @@
+dbt-core~=1.8.0
+dbt-postgres~=1.8.0

constraints/dbt180.txt

@@ -0,0 +1,2 @@
+dbt-core~=1.8.0
+dbt-postgres~=1.8.0

constraints/dbt190-winpy.txt

@@ -0,0 +1,2 @@
+dbt-core~=1.9.0b1
+dbt-postgres~=1.9.0b1

constraints/dbt190.txt

@@ -0,0 +1,2 @@
+dbt-core~=1.9.0b1
+dbt-postgres~=1.9.0b1

docs/Makefile

@@ -20,4 +20,5 @@ help:
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile
+	python generate-auto-docs.py
 	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/generate-auto-docs.py

@@ -0,0 +1,155 @@
+"""Generate some documentation automatically.
+
+This script generates partial documentation sections (i.e. the content of
+`/docs/source/_partials/`) by importing SQLFluff and extracting data about
+rules and dialects.
+
+It should run before every docs generation so that those partial .rst files
+can then be correctly referenced by other sections of the docs. For example
+this file builds the file `/docs/source/_partials/rule_summaries.rst`, which
+is then inserted into `/docs/source/reference/rules.rst` using the directive
+`.. include:: ../_partials/rule_summaries.rst`.
+
+This script is referenced in the `Makefile` and the `make.bat` file to ensure
+it is run at the appropriate moment.
+"""
+
+import json
+from collections import defaultdict
+from pathlib import Path
+
+import sqlfluff
+from sqlfluff.core.plugin.host import get_plugin_manager
+
+base_path = Path(__file__).parent.absolute()
+
+##########################################
+# Generate rule documentation dynamically.
+##########################################
+
+autogen_header = """..
+    NOTE: This file is generated by the conf.py script.
+    Don't edit this by hand
+
+
+"""
+
+table_header = f"""
++{'-' * 42}+{'-' * 50}+{'-' * 30}+{'-' * 20}+
+|{'Bundle' : <42}|{'Rule Name' : <50}|{'Code' : <30}|{'Aliases' : <20}|
++{'=' * 42}+{'=' * 50}+{'=' * 30}+{'=' * 20}+
+"""
+
+# Extract all the rules.
+print("Rule Docs Generation: Reading Rules...")
+rule_bundles = defaultdict(list)
+rule_list = []
+for plugin_rules in get_plugin_manager().hook.get_rules():
+    for rule in plugin_rules:
+        _bundle_name = rule.name.split(".")[0]
+        rule_bundles[_bundle_name].append(rule)
+        rule_list.append((rule.code, rule.name))
+
+# Write them into a json file for use by redirects.
+print("Rule Docs Generation: Writing Rule JSON...")
+with open(base_path / "source/_partials/rule_list.json", "w", encoding="utf8") as f:
+    json.dump(rule_list, f)
+
+# Write them into the table. Bundle by bundle.
+print("Rule Docs Generation: Writing Rule Table...")
+with open(base_path / "source/_partials/rule_table.rst", "w", encoding="utf8") as f:
+    f.write(autogen_header)
+    f.write(table_header)
+    for bundle in sorted(rule_bundles.keys()):
+        # Set the bundle name to the ref.
+        _bundle_name = f":ref:`bundle_{bundle}`"
+        for idx, rule in enumerate(rule_bundles[bundle]):
+            step = 1  # The number of aliases per line.
+            aliases = ", ".join(rule.aliases[:step]) + (
+                "," if len(rule.aliases) > step else ""
+            )
+            name_ref = f":sqlfluff:ref:`{rule.name}`"
+            code_ref = f":sqlfluff:ref:`{rule.code}`"
+            f.write(
+                f"| {_bundle_name : <40} | {name_ref : <48} "
+                f"| {code_ref : <28} | {aliases : <18} |\n"
+            )
+
+            j = 1
+
+            while True:
+                if not rule.aliases[j:]:
+                    break
+                aliases = ", ".join(rule.aliases[j : j + step]) + (
+                    "," if len(rule.aliases[j:]) > step else ""
+                )
+                f.write(f"|{' ' * 42}|{' ' * 50}|{' ' * 30}| {aliases : <18} |\n")
+                j += step
+
+            if idx + 1 < len(rule_bundles[bundle]):
+                f.write(f"|{' ' * 42}+{'-' * 50}+{'-' * 30}+{'-' * 20}+\n")
+            else:
+                f.write(f"+{'-' * 42}+{'-' * 50}+{'-' * 30}+{'-' * 20}+\n")
+            # Unset the bundle name so we don't repeat it.
+            _bundle_name = ""
+    f.write("\n\n")
+
+
+# Write each of the summary files.
+print("Rule Docs Generation: Writing Rule Summaries...")
+with open(base_path / "source/_partials/rule_summaries.rst", "w", encoding="utf8") as f:
+    f.write(autogen_header)
+    for bundle in sorted(rule_bundles.keys()):
+        if "sql" in bundle:
+            # This accounts for things like "TSQL"
+            header_name = bundle.upper()
+        else:
+            header_name = bundle.capitalize()
+        # Write the bundle header.
+        f.write(
+            f".. _bundle_{bundle}:\n\n"
+            f"{header_name} bundle\n"
+            f"{'-' * (len(bundle) + 7)}\n\n"
+        )
+        for rule in rule_bundles[bundle]:
+            f.write(
+                f".. sqlfluff:rule:: {rule.code}\n"
+                f"                   {rule.name}\n\n"
+            )
+            # Separate off the heading so we can bold it.
+            heading, _, doc_body = rule.__doc__.partition("\n")
+            underline_char = '"'
+            f.write(f"    {heading}\n")
+            f.write(f"    {underline_char * len(heading)}\n\n")
+            f.write("    " + doc_body)
+            f.write("\n\n")
+
+print("Rule Docs Generation: Done")
+
+# Extract all the dialects.
+print("Dialect Docs Generation: Reading Dialects...")
+# We make a dictionary of all of them first, because we want to force the ANSI
+# one to be first.
+dialect_dict = {dialect.label: dialect for dialect in sqlfluff.list_dialects()}
+dialect_list = [dialect_dict["ansi"]] + [
+    dialect for dialect_name, dialect in dialect_dict.items() if dialect_name != "ansi"
+]
+
+# Write each of the summary files.
+print("Dialect Docs Generation: Writing Dialect Summaries...")
+with open(
+    base_path / "source/_partials/dialect_summaries.rst", "w", encoding="utf8"
+) as f:
+    f.write(autogen_header)
+    for dialect in dialect_list:
+        f.write(
+            f".. _{dialect.label}_dialect_ref:\n\n"
+            f"{dialect.name}\n{'-' * len(dialect.name)}\n\n"
+            f"**Label**: ``{dialect.label}``\n\n"
+        )
+        if dialect.label != "ansi":
+            f.write(
+                f"**Inherits from**: :ref:`{dialect.inherits_from}_dialect_ref`\n\n"
+            )
+        if dialect.docstring:
+            f.write(dialect.docstring + "\n\n")

docs/make.bat

@@ -25,6 +25,9 @@ if errorlevel 9009 (
 	exit /b 1
 )
 
+REM Generate the rule & dialect docs
+python generate-auto-docs.py
+
 %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
 goto end
 

docs/requirements.txt

@@ -1,5 +1,8 @@
 # Requirements for building docs
 sphinx>=2.4.1
+# 2.13.0 added the sql+jinja syntax
+pygments>=2.13.0
 sphinx-click
+sphinx-reredirects
 doc8
 tqdm

docs/source/_ext/sqlfluff_domain.py

@@ -0,0 +1,147 @@
+"""The sqlfluff domain for documenting rules."""
+
+from sphinx import addnodes
+from sphinx.directives import ObjectDescription
+from sphinx.domains import Domain, ObjType
+from sphinx.roles import XRefRole
+from sphinx.util.nodes import make_refnode
+
+
+class SQLFluffRule(ObjectDescription):
+    """SQLFluff rule directive for sphinx.
+
+    Rule directives can be used as shown below.
+
+    .. code-block:: rst
+
+        .. sqlfluff:rule:: AM01
+                           ambiguous.distinct
+
+            Write the documentation for the rule here.
+
+    To cross reference (i.e. refer to) objects defined like this
+    both the code and name reference is available:
+
+    .. code-block:: rst
+
+        :sqlfluff:ref:`CP02`
+        :sqlfluff:ref:`capitalisation.identifiers`
+
+    """
+
+    def handle_signature(self, sig, signode):
+        """Handle the initial signature of the node.
+
+        This formats the header of the section.
+        """
+        raw_obj_type = "code" if len(sig) == 4 else "rule"
+        obj_type = raw_obj_type.capitalize() + " "
+        signode += addnodes.desc_type(obj_type, obj_type)
+        signode += addnodes.desc_name(sig, sig)
+
+        fullname = obj_type + sig
+        signode["type"] = raw_obj_type
+        signode["sig"] = sig
+        signode["fullname"] = fullname
+        return (fullname, raw_obj_type, sig)
+
+    def add_target_and_index(self, name_cls, sig, signode):
+        """Hook to add the permalink and index entries."""
+        # Add an ID for permalinks
+        node_id = "rule" + "-" + sig
+        signode["ids"].append(node_id)
+        if len(sig) == 4:
+            # If it's a code, add support for legacy links too.
+            # Both of these formats have been used in the past.
+            signode["ids"].append(f"sqlfluff.rules.Rule_{sig}")
+            signode["ids"].append(f"sqlfluff.rules.sphinx.Rule_{sig}")
+        # Add to domain for xref resolution
+        fluff = self.env.get_domain("sqlfluff")
+        fluff.add_rule(sig)
+        # Add to index
+        self.indexnode["entries"].append(("single", sig, node_id, "", None))
+
+    def _object_hierarchy_parts(self, sig_node):
+        return ("bundle", "name")
+
+    def _toc_entry_name(self, sig_node) -> str:
+        # NOTE: toctree unpacking issues are due to incorrectly
+        # setting _toc_parts.
+        sig_node["_toc_parts"] = (
+            "bundle",
+            sig_node["sig"],
+        )
+        if len(sig_node["sig"]) == 4:
+            # It's a code - don't return TOC entry.
+            return ""
+        else:
+            # It's a name
+            return sig_node["sig"]
+
+
+class SQLFluffDomain(Domain):
+    """SQLFluff domain."""
+
+    name = "sqlfluff"
+    label = "sqlfluff"
+
+    object_types = {
+        "rule": ObjType("rule", "rule", "obj"),
+    }
+
+    roles = {
+        "ref": XRefRole(),
+    }
+
+    directives = {
+        "rule": SQLFluffRule,
+    }
+
+    initial_data = {
+        "rules": [],  # object list
+    }
+
+    def get_full_qualified_name(self, node):
+        """Get the fully qualified name of the rule."""
+        return f"rule.{node.arguments[0]}"
+
+    def get_objects(self):
+        """Hook to get all the rules."""
+        yield from self.data["rules"]
+
+    def resolve_xref(self, env, fromdocname, builder, typ, target, node, contnode):
+        """Hook to resolve xrefs.
+
+        References can be made by code or by name, e.g.
+        - :sqlfluff:ref:`LT01`
+        - :sqlfluff:ref:`layout.spacing`
+        """
+        match = [
+            (docname, anchor)
+            for _, sig, _, docname, anchor, _ in self.get_objects()
+            if sig == target
+        ]
+
+        if len(match) > 0:
+            todocname = match[0][0]
+            targ = match[0][1]
+
+            return make_refnode(builder, fromdocname, todocname, targ, contnode, targ)
+        else:
+            print(f"Failed to match xref: {target!r}")
+            return None
+
+    def add_rule(self, signature):
+        """Add a new recipe to the domain."""
+        name = f"rule.{signature}"
+        anchor = f"rule-{signature}"
+
+        # name, dispname, type, docname, anchor, priority
+        self.data["rules"].append(
+            (name, signature, "Rule", self.env.docname, anchor, 0)
+        )
+
+
+def setup(app):
+    """Setup the domain."""
+    app.add_domain(SQLFluffDomain)

docs/source/_partials/.gitignore

@@ -0,0 +1,4 @@
+rule_table.rst
+rule_summaries.rst
+rule_list.json
+dialect_summaries.rst

docs/source/_partials/README.md

@@ -0,0 +1,9 @@
+This folder is ignored from the main build and intended
+only for files included in others via the `.. include::`
+directive.
+
+Some of those files are also auto-generated by scripts,
+in which case they should be included in the `.gitignore`
+and not edited by hand.
+
+See [generate-auto-docs.py](https://github.com/sqlfluff/sqlfluff/blob/main/docs/generate-auto-docs.py) for more info.

docs/source/_partials/starter_config.cfg

@@ -0,0 +1,68 @@
+[sqlfluff]
+
+# Supported dialects https://docs.sqlfluff.com/en/stable/perma/dialects.html
+# Or run 'sqlfluff dialects'
+dialect = snowflake
+
+# One of [raw|jinja|python|placeholder]
+templater = jinja
+
+# Comma separated list of rules to exclude, or None
+# See https://docs.sqlfluff.com/en/stable/perma/rule_disabling.html
+# AM04 (ambiguous.column_count) and ST06 (structure.column_order) are
+# two of the more controversial rules included to illustrate usage.
+exclude_rules = ambiguous.column_count, structure.column_order
+
+# The standard max_line_length is 80 in line with the convention of
+# other tools and several style guides. Many projects however prefer
+# something a little longer.
+# Set to zero or negative to disable checks.
+max_line_length = 120
+
+# CPU processes to use while linting.
+# The default is "single threaded" to allow easy debugging, but this
+# is often undesirable at scale.
+# If positive, just implies number of processes.
+# If negative or zero, implies number_of_cpus - specified_number.
+# e.g. -1 means use all processors but one. 0 means all cpus.
+processes = -1
+
+# If using the dbt templater, we recommend setting the project dir.
+[sqlfluff:templater:dbt]
+project_dir = ./
+
+[sqlfluff:indentation]
+# While implicit indents are not enabled by default. Many of the
+# SQLFluff maintainers do use them in their projects.
+allow_implicit_indents = True
+
+[sqlfluff:rules:aliasing.length]
+min_alias_length = 3
+
+# The default configuration for capitalisation rules is "consistent"
+# which will auto-detect the setting from the rest of the file. This
+# is less desirable in a new project and you may find this (slightly
+# more strict) setting more useful.
+# Typically we find users rely on syntax highlighting rather than
+# capitalisation to distinguish between keywords and identifiers.
+# Clearly, if your organisation has already settled on uppercase
+# formatting for any of these syntax elements then set them to "upper".
+# See https://stackoverflow.com/questions/608196/why-should-i-capitalize-my-sql-keywords-is-there-a-good-reason
+[sqlfluff:rules:capitalisation.keywords]
+capitalisation_policy = lower
+[sqlfluff:rules:capitalisation.identifiers]
+extended_capitalisation_policy = lower
+[sqlfluff:rules:capitalisation.functions]
+extended_capitalisation_policy = lower
+[sqlfluff:rules:capitalisation.literals]
+capitalisation_policy = lower
+[sqlfluff:rules:capitalisation.types]
+extended_capitalisation_policy = lower
+
+# The default configuration for the not equal convention rule is "consistent"
+# which will auto-detect the setting from the rest of the file. This
+# is less desirable in a new project and you may find this (slightly
+# more strict) setting more useful.
+[sqlfluff:rules:convention.not_equal]
+# Default to preferring the "c_style" (i.e. `!=`)
+preferred_not_equal_style = c_style

docs/source/api.rst

@@ -1,36 +0,0 @@
-.. _apiref:
-
-API Reference
-=============
-
-SQLFluff exposes a public api for other python applications to use.
-A basic example of this usage is given here, with the documentation
-for each of the methods below.
-
-
-.. literalinclude:: ../../examples/01_basic_api_usage.py
-   :language: python
-
-
-Simple API commands
--------------------
-
-
-.. automodule:: sqlfluff
-   :members: lint, fix, parse
-
-
-Advanced API usage
-------------------
-
-The simple API presents only a fraction of the functionality present
-within the core SQLFluff library. For more advanced use cases, users
-can import the :code:`Linter()` and :code:`FluffConfig()` classes from
-:code:`sqlfluff.core`. As of version 0.4.0 this is considered as
-*experimental only* as the internals may change without warning in any
-future release. If you come to rely on the internals of SQLFluff, please
-post an issue on GitHub to share what you're up to. This will help shape
-a more reliable, tidy and well documented public API for use.
-
-.. automodule:: sqlfluff.core
-   :members: Linter, Lexer, Parser, FluffConfig

docs/source/architecture.rst

@@ -1,130 +0,0 @@
-Architecture
-============
-
-It is recommended that the following is read in conjunction with exploring
-the codebase. `dialect_ansi.py` in particular is helpful to understand the
-recursive structure of segments and grammars. Some more detail is also given
-on our Wiki_ including a `Contributing Dialect Changes`_ guide.
-
-.. _Wiki: https://github.com/sqlfluff/sqlfluff/wiki/
-.. _`Contributing Dialect Changes`: https://github.com/sqlfluff/sqlfluff/wiki/Contributing-Dialect-Changes
-
-Stage 1, the templater
-----------------------
-
-This stage only applies to templated SQL, most commonly Jinja and dbt. Vanilla
-SQL is sent straight to stage 2, the lexer.
-
-In order to lint templated SQL, SQLFluff must first convert the 'raw' or
-pre-templated code into valid SQL, which can then be parsed. The templater
-returns both the raw and post-templated SQL so that any rule violations which
-occur in templated sections can be ignored and the rest mapped to their
-original line location for user feedback.
-
-.. _Jinja: https://jinja.palletsprojects.com/
-.. _dbt: https://docs.getdbt.com/
-
-*SQLFluff* supports two templating engines: Jinja_ and dbt_.
-
-Under the hood dbt also uses Jinja, but in *SQLFluff* uses a separate
-mechanism which interfaces directly with the dbt python package.
-
-For more details on how to configure the templater see :ref:`templateconfig`.
-
-
-Stage 2, the lexer
-------------------
-
-The lexer takes SQL and separates it into segments of whitespace and
-code. No meaning is imparted; that is the job of the parser.
-
-
-Stage 3, the parser
--------------------
-
-The parser is arguably the most complicated element of SQLFluff, and is
-relied on by all the other elements of the tool to do most of the heavy
-lifting.
-
-#. The lexed segments are parsed using the specified dialect's grammars. In
-   SQLFluff, grammars describe the shape of SQL statements (or their
-   components). The parser attempts to apply each potential grammar to the
-   lexed segments until all the segments have been matched.
-
-#. In SQLFluff, segments form a tree-like structure. The top-level segment is
-   a :code:`FileSegment`, which contains zero or more
-   :code:`StatementSegment`\ s, and so on. Before the segments have been parsed
-   and named according to their type, they are 'raw', meaning they have no
-   classification other than their literal value.
-
-#. The three key components to the parser are segments,
-   :code:`match_grammar`\ s and :code:`parse_grammar`\ s. A segment can be a
-   leaf in the parse tree, such as a :code:`NumericLiteralSegment`, which is
-   simply a number, or can contain many other segments, such as a
-   :code:`SelectStatementSegment`. Each segment can specify a
-   :code:`parse_grammar`, and a :code:`match_grammar`. If both a
-   :code:`match_grammar` and :code:`parse_grammar` are defined in a segment,
-   :code:`match_grammar` is used to quickly prune the tree for branches which
-   do not match segments being parsed, and the :code:`parse_grammar` is then
-   used to refine the branch identified as correct. If only a
-   :code:`match_grammar` is defined, then it serves the purpose of both pruning
-   and refining.
-
-#. A segment's :code:`.parse()` method uses the :code:`parse_grammar`, on
-   which :code:`.match()` is called. The *match* method of this grammar will
-   return a potentially refined structure of the segments within the
-   segment in greater detail. In the example of a :code:`FileSegment`, it
-   first divides up the query into statements and then finishes.
-
-   * *Segments* must implement a :code:`match_grammar`. When :code:`.match()`
-      is called on a segment, this is the grammar which is used to decide
-      whether there is a match.
-
-   * *Grammars* combine *segments* or other *grammars* together in a
-      pre-defined way. For example the :code:`OneOf` grammar will match if any
-      one of its child elements match.
-
-   #. Regardless of whether the :code:`parse_grammar` was used, the next step
-      is to recursively call the :code:`.parse()` method of each of the child
-      segments of the grammar. This operation is wrapped in a method called
-      :code:`.expand()`. In the :code:`FileSegment`, the first step will have
-      transformed a series of raw tokens into :code:`StatementSegment`
-      segments, and the *expand* step will let each of those segments refine
-      the content within them.
-
-   #. During the recursion, the parser eventually reaches segments which have
-      no children (raw segments containing a single token), and so the
-      recursion naturally finishes.
-
-#. If no match is found for a segment, the contents will be wrapped in an
-:code:`UnparsableSegment` which is picked up as a *parsing* error later.
-
-When working on the parser there are a couple of design principles
-to keep in mind.
-
-- Grammars are contained in *dialects*, the root dialect being
-  the *ansi* dialect. The ansi dialect is used to host logic common
-  to all dialects, and so does not necessarily adhere to the formal
-  ansi specification. Other SQL dialects inherit from the ansi dialect,
-  replacing or patching any segments they need to. One reason for the *Ref*
-  grammar is that it allows name resolution of grammar elements at runtime
-  and so a *patched* grammar with some elements overriden can still rely on
-  lower-level elements which haven't been redeclared within the dialect
-- All grammars and segments attempt to match as much as they can and will
-  return partial matches where possible. It is up to the calling grammar
-  or segment to decide whether a partial or complete match is required
-  based on the context it is matching in.
-
-
-Stage 4, the linter
--------------------
-
-Given the complete parse tree, rule classes check for linting errors by
-traversing the tree, looking for segments and patterns of concern. If
-the rule discovers a violation, it returns a :code:`LintResult` pointing
-to the segment which caused the violation.
-
-Some rules are able to *fix* the problems they find. If this is the case,
-the rule will return a list of fixes, which describe changes to be made to
-the tree. This can include edits, inserts, or deletions. Once the fixes
-have been applied, the updated tree is written to the original file.

docs/source/conf.py

@@ -5,28 +5,34 @@ list see the documentation:
 https://www.sphinx-doc.org/en/master/usage/configuration.html
 """
 
-import configparser
+import json
+import os
+import sys
+
+# tomllib is only in the stdlib from 3.11+
+if sys.version_info >= (3, 11):
+    import tomllib
+else:  # pragma: no cover
+    import toml as tomllib
 
 # -- Path setup --------------------------------------------------------------
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-#
-# import os
-# import sys
-# sys.path.insert(0, os.path.abspath('.'))
+
+sys.path.append(os.path.abspath("./_ext"))
 
 # Get the global config info as currently stated
 # (we use the config file to avoid actually loading any python here)
-config = configparser.ConfigParser()
-config.read(["../../setup.cfg"])
-stable_version = config.get("sqlfluff_docs", "stable_version")
+with open("../../pyproject.toml", "rb") as config_file:
+    config = tomllib.load(config_file)
+stable_version = config.get("tool.sqlfluff_docs", "stable_version")
 
 # -- Project information -----------------------------------------------------
 
 project = "SQLFluff"
-copyright = "2019, Alan Cruickshank"
+copyright = "2024, Alan Cruickshank"
 author = "Alan Cruickshank"
 
 # The full version, including alpha/beta/rc tags
@@ -45,6 +51,10 @@ extensions = [
     "sphinx.ext.napoleon",
     # Documenting click commands
     "sphinx_click.ext",
+    # Redirects
+    "sphinx_reredirects",
+    # SQLFluff domain
+    "sqlfluff_domain",
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -53,7 +63,11 @@ templates_path = ["_templates"]
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This pattern also affects html_static_path and html_extra_path.
-exclude_patterns = []
+exclude_patterns = [
+    # Exclude the partials folder, which is made up of files intended
+    # to be included in others.
+    "_partials",
+]
 
 # Master doc
 master_doc = "index"
@@ -86,14 +100,76 @@ html_theme_options = {
     "touch_icon": "images/sqlfluff-sm2-sq.png",
     "github_user": "sqlfluff",
     "github_repo": "sqlfluff",
-    # GitHub Fork button
-    "github_banner": True,
+    # GitHub Fork button (points at a broken link, so disabling it)
+    "github_banner": False,
     # GitHub star button
     "github_type": "star",
     # Use `"true"` instead of `True` for counting GitHub star, see https://ghbtns.com
     "github_count": "true",
-    # Codecov button
-    "codecov_button": True,
+}
+
+# -- Options for redirects ---------------------------------------------
+# https://documatt.gitlab.io/sphinx-reredirects/usage.html
+
+# Load the rule lists to generate rule permalinks
+with open("_partials/rule_list.json", "r") as rule_file:
+    rule_list = json.load(rule_file)
+
+redirects = {
+    # Where there are references to the docs in any of the codebase (whether in
+    # places like the README or in error messages), they should all reference
+    # a perma link (to redirect). This ensures we can support a consistent
+    # link location even if the docs move around.
+    "perma/layout": "../configuration/layout.html",
+    "perma/indent_locations": (
+        "../configuration/layout.html#configuring-indent-locations"
+    ),
+    "perma/hanging_indents": "../configuration/layout.html#hanging-indents",
+    "perma/layout_spacing": (
+        "../configuration/layout.html#configuring-layout-and-spacing"
+    ),
+    "perma/configuration": "../configuration/index.html",
+    "perma/dbt": "../configuration/templating/dbt.html",
+    "perma/cli": "../reference/cli.html",
+    "perma/rules": "../reference/rules.html",
+    "perma/dialects": "../reference/dialects.html",
+    "perma/architecture": "../guides/contributing/architecture.html",
+    "perma/rule_disabling": (
+        "../configuration/rule_configuration.html#enabling-and-disabling-rules"
+    ),
+    "perma/internals": "../reference/internals/index.html",
+    "perma/modularity": "../why_sqlfluff.html#modularity",
+    "perma/indentation": "../configuration/layout.html#configuring-indent-locations",
+    "perma/releasenotes": "../reference/releasenotes.html",
+    "perma/why": "../why_sqlfluff.html",
+    "perma/plugin_dev": "../guides/contributing/plugins.html",
+    "perma/plugin_guide": "../guides/setup/developing_custom_rules.html",
+    "perma/variables": "../configuration/templating/index.html",
+    "perma/python_templating": "../configuration/templating/python.html",
+    "perma/guides": "../guides/index.html",
+    "perma/contribute_dialect_keywords": (
+        "../guides/contributing/dialect.html#dialect_keywords"
+    ),
+    # Add permalinks for rule codes
+    **{
+        f"perma/rule/{code}": (
+            f"../../reference/rules.html#sqlfluff.rules.sphinx.Rule_{code}"
+        )
+        for code, _ in rule_list
+    },
+    # These are legacy links which used to exist in different parts of the
+    # SQLFluff code base, and which we continue to support so those links
+    # aren't dead ends. They should redirect to permalinks.
+    "indentation": "perma/indentation.html",
+    "architecture": "perma/architecture.html",
+    "dialects": "perma/dialects.html",
+    "internals": "perma/internals.html",
+    "layout": "perma/layout.html",
+    "releasenotes": "perma/releasenotes.html",
+    "realworld": "perma/why.html",
+    # This is a legacy link to support older versions of the VSCode plugin.
+    # https://github.com/sqlfluff/vscode-sqlfluff/blob/master/src/features/providers/linter/actions/hover.ts
+    "rules": "perma/rules.html",
 }
 
 

docs/source/configuration.rst

@@ -1,755 +0,0 @@
-.. _config:
-
-Configuration
-=============
-
-SQLFluff accepts configuration either through the command line or
-through configuration files. There is *rough* parity between the
-two approaches with the exception that *templating* configuration
-must be done via a file, because it otherwise gets slightly complicated.
-
-For details of what's available on the command line check out
-the :ref:`cliref`.
-
-For file based configuration *SQLFluff* will look for the following
-files in order. Later files will (if found) will be used to overwrite
-any vales read from earlier files.
-
-- :code:`setup.cfg`
-- :code:`tox.ini`
-- :code:`pep8.ini`
-- :code:`.sqlfluff`
-- :code:`pyproject.toml`
-
-Within these files, the first four will be read like a `cfg file`_, and
-*SQLFluff* will look for sections which start with :code:`sqlfluff`, and where
-subsections are delimited by a semicolon. For example the *jinjacontext*
-section will be indicated in the section started with
-:code:`[sqlfluff:jinjacontext]`.
-
-For example, a snippet from a :code:`.sqlfluff` file (as well as any of the
-supported cfg file types):
-
-.. code-block:: cfg
-
-    [sqlfluff]
-    templater = "jinja"
-    sql_file_exts = ".sql,.sql.j2,.dml,.ddl"
-
-    [sqlfluff:indentation]
-    indented_joins = false
-    indented_using_on = true
-    template_blocks_indent = false
-
-    [sqlfluff:templater]
-    unwrap_wrapped_queries = true
-
-    [sqlfluff:templater:jinja]
-    apply_dbt_builtins = true
-
-For the `pyproject.toml file`_, all valid sections start with
-:code:`tool.sqlfluff` and subsections are delimited by a dot. For example the
-*jinjacontext* section will be indicated in the section started with
-:code:`[tool.sqlfluff.jinjacontext]`.
-
-For example, a snippet from a :code:`pyproject.toml` file:
-
-.. code-block:: cfg
-
-    [tool.sqlfluff.core]
-    templater = "jinja"
-    sql_file_exts = ".sql,.sql.j2,.dml,.ddl"
-
-    [tool.sqlfluff.indentation]
-    indented_joins = false
-    indented_using_on = true
-    template_blocks_indent = false
-
-    [tool.sqlfluff.templater]
-    unwrap_wrapped_queries = true
-
-    [tool.sqlfluff.templater.jinja]
-    apply_dbt_builtins = true
-
-.. _`cfg file`: https://docs.python.org/3/library/configparser.html
-.. _`pyproject.toml file`: https://www.python.org/dev/peps/pep-0518/
-
-Nesting
--------
-
-**SQLFluff** uses **nesting** in its configuration files, with files
-closer *overriding* (or *patching*, if you will) values from other files.
-That means you'll end up with a final config which will be a patchwork
-of all the values from the config files loaded up to that path. The exception
-to this is the value for `templater`, which cannot be set in config files in
-subdirectories of the working directory.
-You don't **need** any config files to be present to make *SQLFluff*
-work. If you do want to override any values though SQLFluff will use
-files in the following locations in order, with values from later
-steps overriding those from earlier:
-
-0. *[...and this one doesn't really count]* There's a default config as
-   part of the SQLFluff package. You can find this below, in the
-   :ref:`defaultconfig` section.
-1. It will look in the user's os-specific app config directory. On OSX this is
-   `~/Library/Preferences/sqlfluff`, Unix is `~/.config/sqlfluff`, Windows is
-   `<home>\\AppData\\Local\\sqlfluff\\sqlfluff`, for any of the filenames
-   above in the main :ref:`config` section. If multiple are present, they will
-   *patch*/*override* each other in the order above.
-2. It will look for the same files in the user's home directory (~).
-3. It will look for the same files in the current working directory.
-4. *[if parsing a file in a subdirectory of the current working directory]*
-   It will look for the same files in every subdirectory between the
-   current working dir and the file directory.
-5. It will look for the same files in the directory containing the file
-   being linted.
-
-This whole structure leads to efficient configuration, in particular
-in projects which utilise a lot of complicated templating.
-
-.. _ruleconfig:
-
-Rule Configuration
-------------------
-
-Rules can be configured with the :code:`.sqlfluff` config files.
-
-Common rule configurations can be set in the :code:`[sqlfluff:rules]` section.
-
-For example:
-
-.. code-block:: cfg
-
-   [sqlfluff:rules]
-   tab_space_size = 4
-   max_line_length = 80
-   indent_unit = space
-   comma_style = trailing
-   allow_scalar = True
-   single_table_references = consistent
-   unquoted_identifiers_policy = all
-
-Rule specific configurations are set in rule specific subsections.
-
-For example, enforce that keywords are upper case by configuring the rule
-:class:`L010 <sqlfluff.core.rules.Rule_L010>`:
-
-.. code-block:: cfg
-
-    [sqlfluff:rules:L010]
-    # Keywords
-    capitalisation_policy = upper
-
-All possible options for rule sections are documented in :ref:`ruleref`.
-
-For an overview of the most common rule configurations that you may want to
-tweak, see `Default Configuration`_ (and use :ref:`ruleref` to find the
-available alternatives).
-
-Enabling and Disabling Rules
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-To disable individual rules, set :code:`exclude_rules` in the top level section
-of sqlfluff configuration. The value is a comma separated list of rule ids.
-
-For example, to disable the rules :class:`L022 <sqlfluff.core.rules.Rule_L022>`
-and :class:`L027 <sqlfluff.core.rules.Rule_L027>`:
-
-.. code-block:: cfg
-
-    [sqlfluff]
-    exclude_rules = L022, L027
-
-To enable individual rules, configure :code:`rules`, respectively.
-
-For example, to enable :class:`L027 <sqlfluff.core.rules.Rule_L027>`:
-
-.. code-block:: cfg
-
-    [sqlfluff]
-    rules = L027
-
-If both :code:`exclude_rules` and :code:`rules` have non-empty value, then the
-excluded rules are removed from the rules list. This allows for example
-enabling common rules on top level but excluding some on subdirectory level.
-
-Additionally, some rules have a special :code:`force_enable` configuration
-option, which allows to enable the given rule even for dialects where it is
-disabled by default. The rules that support this can be found in the
-:ref:`ruleref`.
-
-The default values can be seen in `Default Configuration`_.
-
-See also: `Ignoring Errors & Files`_.
-
-.. _templateconfig:
-
-Jinja Templating Configuration
-------------------------------
-
-When thinking about Jinja templating there are two different kinds of things
-that a user might want to fill into a templated file, *variables* and
-*functions/macros*. Currently *functions* aren't implemented in any
-of the templaters.
-
-Variable Templating
-^^^^^^^^^^^^^^^^^^^
-
-Variables are available in the *jinja*, *python* and *placeholder* templaters.
-By default the templating engine will expect variables for templating to be
-available in the config, and the templater will be look in the section
-corresponding to the context for that templater. By convention, the config for
-the *jinja* templater is found in the *sqlfluff:templater:jinja:context
-section, the config for the *python* templater is found in the
-*sqlfluff:templater:python:context* section, the one for the *placeholder*
-templater is found in the *sqlfluff:templater:placeholder:context* section
-
-For example, if passed the following *.sql* file:
-
-.. code-block:: jinja
-
-    SELECT {{ num_things }} FROM {{ tbl_name }} WHERE id > 10 LIMIT 5
-
-...and the following configuration in *.sqlfluff* in the same directory:
-
-.. code-block:: cfg
-
-    [sqlfluff:templater:jinja:context]
-    num_things=456
-    tbl_name=my_table
-
-...then before parsing, the sql will be transformed to:
-
-.. code-block:: sql
-
-    SELECT 456 FROM my_table WHERE id > 10 LIMIT 5
-
-.. note::
-
-    If there are variables in the template which cannot be found in
-    the current configuration context, then this will raise a `SQLTemplatingError`
-    and this will appear as a violation without a line number, quoting
-    the name of the variable that couldn't be found.
-
-Placeholder templating
-^^^^^^^^^^^^^^^^^^^^^^
-
-Libraries such as SQLAlchemy or Psycopg use different parameter placeholder
-styles to mark where a parameter has to be inserted in the query.
-
-For example a query in SQLAlchemy can look like this:
-
-.. code-block:: sql
-
-    SELECT * FROM table WHERE id = :myid
-
-At runtime `:myid` will be replace by a value provided by the application and
-escaped as needed, but this is not standard SQL and cannot be parsed as is.
-
-In order to parse these queries is then necessary to replace these
-placeholders with sample values, and this is done with the placeholder
-templater.
-
-Placeholder templating can be enabled in the config using:
-
-.. code-block:: cfg
-
-    [sqlfluff]
-    templater = placeholder
-
-A few common styles are supported:
-
-.. code-block:: sql
-   :force:
-
-    -- colon
-    WHERE bla = :my_name
-
-    -- numeric_colon
-    WHERE bla = :2
-
-    -- pyformat
-    WHERE bla = %(my_name)s
-
-    -- dollar
-    WHERE bla = $my_name
-
-    -- question_mark
-    WHERE bla = ?
-
-    -- numeric_dollar
-    WHERE bla = $3
-
-    -- percent
-    WHERE bla = %s
-
-    -- ampersand
-    WHERE bla = &s or WHERE bla = &{s} or USE DATABASE MARK_{ENV}
-
-These can be configured by setting `param_style` to the names above:
-
-.. code-block:: cfg
-
-    [sqlfluff:templater:placeholder]
-    param_style = colon
-    my_name = 'john'
-
-then it is necessary to set sample values for each parameter, like `my_name`
-above. Notice that the value needs to be escaped as it will be replaced as a
-string during parsing.
-
-When parameters are positional, like `question_mark`, then their name is
-simply the order in which they appear, starting with `1`.
-
-.. code-block:: cfg
-
-    [sqlfluff:templater:placeholder]
-    param_style = question_mark
-    1 = 'john'
-
-In case you need a parameter style different from the ones above, you can pass
-a custom regex.
-
-.. code-block:: cfg
-
-    [sqlfluff:templater:placeholder]
-    param_regex = __(?P<param_name>[\w_]+)__
-    my_name = 'john'
-
-N.B. quotes around `param_regex` in the config are
-interpreted literally by the templater.
-e.g. `param_regex='__(?P<param_name>[\w_]+)__'` matches
-`'__some_param__'` not `__some_param__`
-
-the named parameter `param_name` will be used as the key to replace, if
-missing, the parameter is assumed to be positional and numbers are used insead.
-
-Also consider making a pull request to the project to have your style added,
-it may be useful to other people and simplify your configuration.
-
-Complex Variable Templating
-^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Two more advanced features of variable templating are *case sensitivity*
-and *native python types*. Both are illustrated in the following example:
-
-.. code-block:: cfg
-
-    [sqlfluff:templater:jinja:context]
-    my_list = ['a', 'b', 'c']
-    MY_LIST = ("d", "e", "f")
-    my_where_dict = {"field_1": 1, "field_2": 2}
-
-.. code-block:: jinja
-
-    SELECT
-        {% for elem in MY_LIST %}
-            '{{elem}}' {% if not loop.last %}||{% endif %}
-        {% endfor %} as concatenated_list
-    FROM tbl
-    WHERE
-        {% for field, value in my_where_dict.items() %}
-            {{field}} = {{value}} {% if not loop.last %}and{% endif %}
-        {% endfor %}
-
-...will render as...
-
-.. code-block:: sql
-
-    SELECT
-        'd' || 'e' || 'f' as concatenated_list
-    FROM tbl
-    WHERE
-        field_1 = 1 and field_2 = 2
-
-Note that the variable was replaced in a case sensitive way and that the
-settings in the config file were interpreted as native python types.
-
-Macro Templating
-^^^^^^^^^^^^^^^^
-
-Macros (which also look and feel like *functions* are available only in the
-*jinja* templater. Similar to `Variable Templating`_, these are specified in
-config files, what's different in this case is how they are named. Similar to
-the *context* section above, macros are configured separately in the *macros*
-section of the config. Consider the following example.
-
-If passed the following *.sql* file:
-
-.. code-block:: jinja
-
-    SELECT {{ my_macro(6) }} FROM some_table
-
-...and the following configuration in *.sqlfluff* in the same directory (note
-the tight control of whitespace):
-
-.. code-block:: cfg
-
-    [sqlfluff:templater:jinja:macros]
-    a_macro_def = {% macro my_macro(n) %}{{ n }} + {{ n * 2 }}{% endmacro %}
-
-...then before parsing, the sql will be transformed to:
-
-.. code-block:: sql
-
-    SELECT 6 + 12 FROM some_table
-
-Note that in the code block above, the variable name in the config is
-*a_macro_def*, and this isn't apparently otherwise used anywhere else.
-Broadly this is accurate, however within the configuration loader this will
-still be used to overwrite previous *values* in other config files. As such
-this introduces the idea of config *blocks* which could be selectively
-overwritten by other configuration files downstream as required.
-
-In addition to macros specified in the config file, macros can also be
-loaded from files or folders. This is specified in the config file:
-
-.. code-block:: cfg
-
-    [sqlfluff:templater:jinja]
-    load_macros_from_path = my_macros
-
-`load_macros_from_path` is a comma-separated list of files or folders. SQLFluff
-will load macros from any :code:`.sql` file found in the specified locations.
-Locations are *relative to the config file*. For example, if the config file
-above was found at :code:`/home/my_project/.sqlfluff` then SQLFluff will look
-for macros in the folder :code:`/home/my_project/my_macros/` (but not
-subfolders). Alternatively, the path can also be a :code:`.sql` itself. Any
-macros defined in the config will always take precedence over a macro defined
-in the path.
-
-**Note:** The `load_macros_from_path` also defines the search path for Jinja
-[include](https://jinja.palletsprojects.com/en/3.0.x/templates/#include) or
-[import](https://jinja.palletsprojects.com/en/3.0.x/templates/#import).
-Unlike with macros (as noted above), subdirectories are supported. For example,
-if `load_macros_from_path` is set to `my_macros`, and there is a file
-`my_macros/subdir/my_file.sql`, you can do:
-
-.. code-block:: jinja
-
-   {% include 'subdir/include_comment.sql' %}
-
-.. note::
-
-    Throughout the templating process **whitespace** will still be treated
-    rigorously, and this includes **newlines**. In particular you may choose
-    to provide *dummy* macros in your configuration different from the actual
-    macros used in production.
-
-    **REMEMBER:** The reason SQLFluff supports macros is to *enable* it to parse
-    templated sql without it being a blocker. It shouldn't be a requirement that
-    the *templating* is accurate - it only needs to work well enough that
-    *parsing* and *linting* are helpful.
-
-Builtin Macro Blocks
-^^^^^^^^^^^^^^^^^^^^
-
-One of the main use cases which inspired *SQLFluff* as a project was `dbt`_.
-It uses jinja templating extensively and leads to some users maintaining large
-repositories of sql files which could potentially benefit from some linting.
-
-.. note::
-    *SQLFluff* has now a tighter integration with dbt through the "dbt" templater.
-    It is the recommended templater for dbt projects. If used, it eliminates the
-    need for the overrides described in this section.
-
-    To use the dbt templater, go to `dbt Project Configuration`_.
-
-*SQLFluff* anticipates this use case and provides some built in macro blocks
-in the `Default Configuration`_ which assist in getting started with `dbt`_
-projects. In particular it provides mock objects for:
-
-* *ref*: The mock version of this provided simply returns the model reference
-  as the name of the table. In most cases this is sufficient.
-* *config*: A regularly used macro in `dbt`_ to set configuration values. For
-  linting purposes, this makes no difference and so the provided macro simply
-  returns nothing.
-
-.. note::
-    If there are other builtin macros which would make your life easier,
-    consider submitting the idea (or even better a pull request) on `github`_.
-
-.. _`dbt`: https://www.getdbt.com/
-.. _`github`: https://www.github.com/sqlfluff/sqlfluff
-
-Library Templating
-^^^^^^^^^^^^^^^^^^
-
-If using *SQLFluff* for dbt with jinja as your templater, you may have library
-function calls within your sql files that can not be templated via the
-normal macro templating mechanisms:
-
-.. code-block:: jinja
-
-    SELECT foo, bar FROM baz {{ dbt_utils.group_by(2) }}
-
-To template these libraries, you can use the `sqlfluff:jinja:library_path`
-config option:
-
-.. code-block:: cfg
-
-    [sqlfluff:templater:jinja]
-    library_path = sqlfluff_libs
-
-This will pull in any python modules from that directory and allow sqlfluff
-to use them in templates. In the above example, you might define a file at
-`sqlfluff_libs/dbt_utils.py` as:
-
-.. code-block:: python
-
-    def group_by(n):
-        return "GROUP BY 1,2"
-
-
-If an `__init__.py` is detected, it will be loaded alongside any modules and
-submodules found within the library path.
-
-.. code-block:: jinja
-
-   SELECT
-      {{ custom_sum('foo', 'bar') }},
-      {{ foo.bar.another_sum('foo', 'bar') }}
-   FROM
-      baz
-
-`sqlfluff_libs/__init__.py`:
-
-.. code-block:: python
-
-    def custom_sum(a: str, b: str) -> str:
-        return a + b
-
-`sqlfluff_libs/foo/__init__.py`:
-
-.. code-block:: python
-
-    # empty file
-
-`sqlfluff_libs/foo/bar.py`:
-
-.. code-block:: python
-
-     def another_sum(a: str, b: str) -> str:
-        return a + b
-
-dbt Project Configuration
--------------------------
-
-.. note::
-    From sqlfluff version 0.7.0 onwards, the dbt templater has been moved
-    to a separate plugin and python package. Projects that were already using
-    the dbt templater may initially fail after an upgrade to 0.7.0+. See the
-    installation instructions below to install the dbt templater.
-
-    dbt templating is still a relatively new feature added in 0.4.0 and
-    is still in very active development! If you encounter an issue, please
-    let us know in a GitHub issue or on the SQLFluff slack workspace.
-
-dbt is not the default templater for *SQLFluff* (it is Jinja). For using
-*SQLFluff* with a dbt project, users can either use the `jinja` templater
-(which may be slightly faster, but will not support the full spectrum of
-macros) or the `dbt` templater, which uses dbt itself to render the
-sql (meaning that there is a much more reliable representation of macros,
-but a potential performance hit accordingly). At this stage we recommend
-that users try both approaches and choose according to the method that
-they intend to use *SQLFluff*.
-
-A simple rule of thumb might be:
-
-- If you are using *SQLFluff* in a CI/CD context, where speed is not
-  critical but accuracy in rendering sql is, then the `dbt` templater
-  may be more appropriate.
-- If you are using *SQLFluff* in an IDE or on a git hook, where speed
-  of response may be more important, then the `jinja` templater may
-  be more appropriate.
-
-Installation & Configuration
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-In order to get started using *SQLFluff* with a dbt project you will
-first need to install the relevant `dbt adapter`_ for your dialect
-and the :code:`sqlfluff-templater-dbt` package using
-your package manager of choice (e.g.
-:code:`pip install dbt-postgres sqlfluff-templater-dbt`) and then will need the
-following configuration:
-
-.. _`dbt adapter`: https://docs.getdbt.com/docs/available-adapters
-
-In *.sqlfluff*:
-
-.. code-block:: cfg
-
-    [sqlfluff]
-    templater = dbt
-
-In *.sqlfluffignore*:
-
-.. code-block:: text
-
-    target/
-    # dbt <1.0.0
-    dbt_modules/
-    # dbt >=1.0.0
-    dbt_packages/
-    macros/
-
-You can set the dbt project directory, profiles directory and profile with:
-
-.. code-block:: cfg
-
-    [sqlfluff:templater:dbt]
-    project_dir = <relative or absolute path to dbt_project directory>
-    profiles_dir = <relative or absolute path to the directory that contains the profiles.yml file>
-    profile = <dbt profile>
-    target = <dbt target>
-
-.. note::
-
-    If the `profiles_dir` setting is omitted, SQLFluff will look for the profile
-    in the default location, which varies by operating system. On Unix-like
-    operating systems (e.g. Linux or macOS), the default profile directory is
-    `~/.dbt/`. On Windows, you can determine your default profile directory by
-    running `dbt debug --config-dir`.
-
-Known Caveats
-^^^^^^^^^^^^^
-
-- To use the dbt templater, you must set `templater = dbt` in the `.sqlfluff`
-  config file in the directory where sqlfluff is run. The templater cannot
-  be changed in `.sqlfluff` files in subdirectories.
-- In SQLFluff 0.4.0 using the dbt templater requires that all files
-  within the root and child directories of the dbt project must be part
-  of the project. If there are deployment scripts which refer to SQL files
-  not part of the project for instance, this will result in an error.
-  You can overcome this by adding any non-dbt project SQL files to
-  .sqlfluffignore.
-
-
-CLI Arguments
--------------
-
-You already know you can pass arguments (:code:`--verbose`,
-:code:`--exclude-rules`, etc.) through the CLI commands (:code:`lint`,
-:code:`fix`, etc.):
-
-.. code-block:: text
-
-    $ sqlfluff lint my_code.sql -v --exclude-rules L022,L027
-
-You might have arguments that you pass through every time, e.g rules you
-*always* want to ignore. These can also be configured:
-
-.. code-block:: cfg
-
-    [sqlfluff]
-    verbose = 1
-    exclude_rules = L022,L027
-
-Note that while the :code:`exclude_rules` config looks similar to the
-above example, the :code:`verbose` config has an integer value. This is
-because :code:`verbose` is *stackable* meaning there are multiple levels
-of verbosity that are available for configuration. See :ref:`cliref` for
-more details about the available CLI arguments. For more details about rule
-exclusion, see `Enabling and Disabling Rules`_.
-
-Ignoring Errors & Files
------------------------
-
-Ignoring individual lines
-^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Similar to `flake8's ignore`_, individual lines can be ignored by adding
-:code:`-- noqa` to the end of the line. Additionally, specific rules can
-be ignored by quoting their code or the category.
-
-.. code-block:: sql
-
-    -- Ignore all errors
-    SeLeCt  1 from tBl ;    -- noqa
-
-    -- Ignore rule L014 & rule L030
-    SeLeCt  1 from tBl ;    -- noqa: L014,L030
-
-    -- Ignore all parsing errors
-    SeLeCt from tBl ;       -- noqa: PRS
-
-.. _`flake8's ignore`: https://flake8.pycqa.org/en/3.1.1/user/ignoring-errors.html#in-line-ignoring-errors
-
-Ignoring line ranges
-^^^^^^^^^^^^^^^^^^^^
-
-Similar to `pylint's "pylint" directive"`_, ranges of lines can be ignored by
-adding :code:`-- noqa:disable=<rule>[,...] | all` to the line. Following this
-directive, specified rules (or all rules, if "all" was specified) will be
-ignored until a corresponding `-- noqa:enable=<rule>[,...] | all` directive.
-
-.. code-block:: sql
-
-    -- Ignore rule L012 from this line forward
-    SELECT col_a a FROM foo -- noqa: disable=L012
-
-    -- Ignore all rules from this line forward
-    SELECT col_a a FROM foo -- noqa: disable=all
-
-    -- Enforce all rules from this line forward
-    SELECT col_a a FROM foo -- noqa: enable=all
-
-
-.. _`pylint's "pylint" directive"`: http://pylint.pycqa.org/en/latest/user_guide/message-control.html
-
-.. _sqlfluffignore:
-
-Ignoring types of errors
-^^^^^^^^^^^^^^^^^^^^^^^^
-General *categories* of errors can be ignored using the ``--ignore`` command
-line option or the ``ignore`` setting in ``.sqlfluffignore``. Types of errors
-that can be ignored include:
-
-* ``lexing``
-* ``linting``
-* ``parsing``
-* ``templating``
-
-.sqlfluffignore
-^^^^^^^^^^^^^^^
-
-Similar to `Git's`_ :code:`.gitignore` and `Docker's`_ :code:`.dockerignore`,
-SQLFluff supports a :code:`.sqlfluffignore` file to control which files are and
-aren't linted. Under the hood we use the python `pathspec library`_ which also
-has a brief tutorial in their documentation.
-
-An example of a potential :code:`.sqlfluffignore` placed in the root of your
-project would be:
-
-.. code-block:: cfg
-
-    # Comments start with a hash.
-
-    # Ignore anything in the "temp" path
-    /temp/
-
-    # Ignore anything called "testing.sql"
-    testing.sql
-
-    # Ignore any ".tsql" files
-    *.tsql
-
-Ignore files can also be placed in subdirectories of a path which is being
-linted and the sub files will also be applied within that subdirectory.
-
-
-.. _`Git's`: https://git-scm.com/docs/gitignore#_pattern_format
-.. _`Docker's`: https://docs.docker.com/engine/reference/builder/#dockerignore-file
-.. _`pathspec library`: https://python-path-specification.readthedocs.io/
-
-.. _defaultconfig:
-
-Default Configuration
----------------------
-
-The default configuration is as follows, note the `Builtin Macro Blocks`_ in
-section *[sqlfluff:templater:jinja:macros]* as referred to above.
-
-.. literalinclude:: ../../src/sqlfluff/core/default_config.cfg
-   :language: cfg
-   :linenos:

docs/source/configuration/default_configuration.rst

@@ -0,0 +1,36 @@
+.. _defaultconfig:
+
+Default Configuration
+---------------------
+
+The default configuration is as follows, note the :ref:`builtin_jinja_blocks`
+in section :code:`[sqlfluff:templater:jinja:macros]` as referred to above.
+
+.. note::
+
+    This shows the *entire* default config. **We do not recommend that users**
+    **copy this whole config as the starter config file for their project**.
+
+    This is for two reasons:
+
+    #. The config file should act as a form of *documentation* for your team.
+       A record of what decisions you've made which govern how your format your
+       sql. By having a more concise config file, and only defining config settings
+       where they differ from the defaults - you are more clearly stating to your
+       team what choices you've made.
+
+    #. As the project evolves, the structure of the config file may change
+       and we will attempt to make changes as backward compatible as possible.
+       If you have not overridden a config setting in your project, we can
+       easily update the default config to match your expected behaviour over time.
+       We may also find issues with the default config which we can also fix
+       in the background. *However*, the longer your local config file, the
+       more work it will be to update and migrate your config file between
+       major versions.
+
+    If you are starting a fresh project and are looking for a good *starter config*,
+    check out the :ref:`starter_config` section above.
+
+
+.. literalinclude:: ../../../src/sqlfluff/core/default_config.cfg
+   :language: cfg

docs/source/configuration/ignoring_configuration.rst

@@ -0,0 +1,100 @@
+.. _ignoreconfig:
+
+Ignoring Errors & Files
+-----------------------
+
+.. _inline_ignoring_errors:
+
+Ignoring individual lines
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Similar to `flake8's ignore`_, individual lines can be ignored by adding
+:code:`-- noqa` to the end of the line. Additionally, specific rules can
+be ignored by quoting their code or the category.
+
+.. code-block:: sql
+
+    -- Ignore all errors
+    SeLeCt  1 from tBl ;    -- noqa
+
+    -- Ignore rule CP02 & rule CP03
+    SeLeCt  1 from tBl ;    -- noqa: CP02,CP03
+
+    -- Ignore all parsing errors
+    SeLeCt from tBl ;       -- noqa: PRS
+
+.. note::
+   It should be noted that ignoring ``TMP`` and ``PRS`` errors can lead to
+   incorrect ``sqlfluff lint`` and ``sqfluff fix`` results as `SQLFluff` can
+   misinterpret the SQL being analysed.
+
+.. _`flake8's ignore`: https://flake8.pycqa.org/en/3.1.1/user/ignoring-errors.html#in-line-ignoring-errors
+
+.. _inline_ignoring_ranges:
+
+Ignoring line ranges
+^^^^^^^^^^^^^^^^^^^^
+
+Similar to `pylint's "pylint" directive"`_, ranges of lines can be ignored by
+adding :code:`-- noqa:disable=<rule>[,...] | all` to the line. Following this
+directive, specified rules (or all rules, if "all" was specified) will be
+ignored until a corresponding `-- noqa:enable=<rule>[,...] | all` directive.
+
+.. code-block:: sql
+
+    -- Ignore rule AL02 from this line forward
+    SELECT col_a a FROM foo -- noqa: disable=AL02
+
+    -- Ignore all rules from this line forward
+    SELECT col_a a FROM foo -- noqa: disable=all
+
+    -- Enforce all rules from this line forward
+    SELECT col_a a FROM foo -- noqa: enable=all
+
+
+.. _`pylint's "pylint" directive"`: http://pylint.pycqa.org/en/latest/user_guide/message-control.html
+
+.. _sqlfluffignore:
+
+:code:`.sqlfluffignore`
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Similar to `Git's`_ :code:`.gitignore` and `Docker's`_ :code:`.dockerignore`,
+SQLFluff supports a :ref:`sqlfluffignore` file to control which files are and
+aren't linted. Under the hood we use the python `pathspec library`_ which also
+has a brief tutorial in their documentation.
+
+An example of a potential :ref:`sqlfluffignore` placed in the root of your
+project would be:
+
+.. code-block:: cfg
+
+    # Comments start with a hash.
+
+    # Ignore anything in the "temp" path
+    /temp/
+
+    # Ignore anything called "testing.sql"
+    testing.sql
+
+    # Ignore any ".tsql" files
+    *.tsql
+
+Ignore files can also be placed in subdirectories of a path which is being
+linted and the sub files will also be applied within that subdirectory.
+
+
+.. _`Git's`: https://git-scm.com/docs/gitignore#_pattern_format
+.. _`Docker's`: https://docs.docker.com/engine/reference/builder/#dockerignore-file
+.. _`pathspec library`: https://python-path-specification.readthedocs.io/
+
+Ignoring types of errors
+^^^^^^^^^^^^^^^^^^^^^^^^
+General *categories* of errors can be ignored using the ``--ignore`` command
+line option or the ``ignore`` setting in :ref:`sqlfluffignore`. Types of errors
+that can be ignored include:
+
+* :code:`lexing`
+* :code:`linting`
+* :code:`parsing`
+* :code:`templating`

docs/source/configuration/index.rst

@@ -0,0 +1,14 @@
+.. _config:
+
+Configuration
+=============
+
+.. toctree::
+   :maxdepth: 2
+
+   setting_configuration
+   rule_configuration
+   layout
+   templating/index
+   ignoring_configuration
+   default_configuration

docs/source/configuration/layout.rst

@@ -0,0 +1,769 @@
+.. _layoutref:
+
+Layout & Whitespace Configuration
+=================================
+
+If there is one part of building a linter that is going to be controversial
+it's going to be **whitespace** (closely followed by **cApiTaLiSaTiOn** 😁).
+
+More specifically, **whitespace** divides into three key themes:
+
+#. **Spacing**: The amount of whitespace between elements on the same line.
+
+#. **Line Breaks**: The choice of where within the code it is inappropriate,
+   appropriate or even compulsory to have a line break.
+
+#. **Indentation**: Given a line break, how much whitespace should precede
+   the first code element on that line.
+
+*SQLFluff* aims to be *opinionated* on this theme, but also *configurable*
+(see :ref:`layoutconfig`). The tool will have a default viewpoint and will aim
+to have views on all of the important aspects of SQL layout, but if you
+(or your organisation) don't like those views then we aim to allow enough
+configuration that you can lint in line with your views, and still use
+*SQLFluff*. For more information on how to configure rules to your own
+viewpoint see :ref:`config`.
+
+.. note::
+
+    This section of the docs handles the intent and reasoning behind how
+    layout is handled by SQLFluff. For a deeper look at how this is achieved
+    internally see :ref:`reflowinternals`.
+
+
+Spacing
+-------
+
+Of the different elements of whitespace, spacing is likely the least
+controversial. By default, all elements are separated by a single space
+character. Except for very specific circumstances (see section on
+:ref:`alignedelements`), any additional space between elements is
+usually unwanted and a distraction for the reader. There are however
+several common cases where *no whitespace* is more appropriate, which
+fall into two cases (for more details on where to configure these see
+:ref:`layoutspacingconfig`).
+
+#. *No whitespace but a newline is allowed.* This option is configured
+   using the :code:`touch` option in the :code:`spacing_*` configuration
+   settings. The most common example of this is the spacing around commas.
+   For example :code:`SELECT a , b` would be unusual and more normally be
+   written :code:`SELECT a, b`. Inserting a newline between the :code:`a`
+   and comma would not cause issues and may even be desired, for example:
+
+   .. code-block:: sql
+
+      SELECT
+         col_a
+         , col_b
+         -- Newline present before column
+         , col_c
+         -- When inline, comma should still touch element before.
+         , GREATEST(col_d, col_e) as col_f
+      FROM tbl_a
+
+#. *No whitespace and a newline is not allowed.* This option is
+   configured using the :code:`inline` option in the :code:`spacing_*`
+   configuration settings. The most common example of this is spacing
+   within the parts of qualified identifier e.g. :code:`my_schema.my_table`.
+   If a newline were present between the :code:`.` and either
+   :code:`my_schema` or :code:`my_table`, then the expression would not
+   parse and so no newlines should be allowed.
+
+
+.. _alignedelements:
+
+Aligned elements
+^^^^^^^^^^^^^^^^
+
+A special case of spacing is where elements are set to be aligned
+within some limits. This is not enabled by default, but can be
+be configured to achieve layouts like:
+
+.. code-block:: sql
+
+   SELECT
+      a           AS first_column,
+      b           AS second_column,
+      (a + b) / 2 AS third_column
+   FROM foo AS bar
+
+In this example, the alias expressions are all aligned with each other.
+To configure this, SQLFluff needs to know what elements to
+align and how far to search to find elements which should be aligned
+with each other. The configuration to achieve this layout is:
+
+.. code-block:: ini
+
+   [sqlfluff:layout:type:alias_expression]
+   # We want non-default spacing _before_ the alias expressions.
+   spacing_before = align
+   # We want to align them within the next outer select clause.
+   # This means for example that alias expressions within the FROM
+   # or JOIN clause would _not_ be aligned with them.
+   align_within = select_clause
+   # The point at which to stop searching outward for siblings, which
+   # in this example would likely be the boundary of a CTE. Stopping
+   # when we hit brackets is usually a good rule of thumb for this
+   # configuration.
+   align_scope = bracketed
+
+Of these configuration values, the :code:`align_scope` is potentially
+the least obvious. The following example illustrates the impact it has.
+
+.. code-block:: sql
+
+   -- With
+   --    align_scope = bracketed
+   --    align_within = select_clause
+
+   WITH foo as (
+      SELECT
+         a,
+         b,
+         c     AS first_column
+         d + e AS second_column
+   )
+
+   SELECT
+      a           AS first_column,
+      (a + b) / 2 AS third_column
+   FROM foo AS bar;
+
+   -- With
+   --    align_scope = bracketed
+   --    align_within = statement
+
+   WITH foo as (
+      SELECT
+         a,
+         b,
+         c     AS first_column
+         d + e AS second_column
+   )
+
+   SELECT
+      a           AS first_column,
+      (a + b) / 2 AS third_column
+   FROM foo       AS bar            -- Now the FROM alias is also aligned.
+
+   -- With
+   --    align_scope = file
+   --    align_within = select_clause
+
+   WITH foo as (
+      SELECT
+         a,
+         b,
+         c        AS first_column   -- Now the aliases here are aligned
+         d + e    AS second_column  -- with the outer query.
+   )
+
+   SELECT
+      a           AS first_column,
+      (a + b) / 2 AS third_column
+   FROM foo AS bar
+
+   -- With
+   --    align_scope = file
+   --    align_within = statement
+
+   WITH foo as (
+      SELECT
+         a,
+         b,
+         c        AS first_column
+         d + e    AS second_column
+   )
+
+   SELECT
+      a           AS first_column,
+      (a + b) / 2 AS third_column
+   FROM foo       AS bar
+
+
+Line Breaks
+-----------
+
+When controlling line breaks, we are trying to achieve a few different things:
+
+#. Do we have *enough* line breaks that *line length* doesn't become
+   excessive. Long lines are hard to read, especially given that readers
+   may be on varying screen sizes or have multiple windows open. This is
+   (of course) configurable, but the default is 80 characters (in line with
+   the `dbt Labs SQL style guide`_.)
+
+#. Is the positioning of *blank lines* (i.e. lines with nothing other
+   than whitespace on them) appropriate. There are some circumstances
+   where a blank line is *desired* (e.g. between CTEs). There are others
+   where they are not, in particular *multiple blank lines*, for example
+   at the beginning of a file.
+
+#. Where we do have line breaks, are they positioned appropriately and
+   consistently with regards to other elements around them. This is most
+   common when it comes to *commas*, and whether they should be *leading*
+   (e.g. :code:`, my_column`) or *trailing* (e.g. :code:`my_column,`). In
+   less common cases, it may also be desirable for some elements to have both
+   a line break *before and after* (e.g. a set operator such as `UNION`).
+
+
+Indentation
+-----------
+
+Lastly, given we have multiple lines of SQL, to what extent should we indent
+some lines to provide visual cues to the structure of that SQL. It's
+important to note that SQL is *not* whitespace sensitive in its
+interpretation and that means that any principles we apply here are entirely
+for the benefit of humans. *Your database doesn't care*.
+
+The indentation therefore should be treated as a *hint* to the reader of
+the structure of the code. This explains the common practice within most
+languages that nested elements (for example the contents of a set of brackets
+in a function call) should be indented one step from the outer elements. It's
+also convention that elements *with the same level* in a nested structure
+should have *the same indentation*, at least with regards to their local
+surroundings. As an example:
+
+.. code-block:: sql
+
+   SELECT
+      nested_within_select AS first_column,
+      some_function(
+         nested_within_function,
+         also_nested_within_function
+      ) AS indented_the_same_as_opening_bracket
+   FROM indented_the_same_as_select
+
+Comment Indents
+^^^^^^^^^^^^^^^
+
+.. note::
+
+      The notes here about block comments are not implemented prior
+      to 2.0.x. They should be coming in that release or soon after.
+
+
+**Comments** are dealt with differently, depending on whether they're
+*block* comments (:code:`/* like this */`), which might optionally
+include newlines, or *inline* comments (:code:`-- like this`) which
+are necessarily only on one line.
+
+*  *Block comments* cannot share a line with any code elements (so
+   in effect they must start on their own new line), they cannot be
+   followed by any code elements on the same line (and so in effect
+   must be followed by a newline, if we are to avoid trailing
+   whitespace). None of the lines within the block comment may have
+   an indent less than the first line of the block comment (although
+   additional indentation within a comment is allowed), and that first
+   line should be aligned with the first code element *following*
+   the block comment.
+
+   .. code-block:: sql
+
+      SELECT
+         /* This is a block comment starting on a new line
+         which contains a newline (continuing with at least
+         the same indent.
+            - potentially containing greater indents
+            - having no other code following it in the same line
+            - and aligned with the line of code following it */
+         this_column as what_we_align_the_column_to
+      FROM my_table
+
+*  *Inline comments* can be on the same line as other code, but are
+   subject to the same line-length restrictions. If they don't fit
+   on the same line (or if it just looks nicer) they can also be
+   the only element on a line. In this latter case, they should be
+   aligned with the first code element *following* the comment.
+
+   .. code-block:: sql
+
+      SELECT
+         -- This is fine
+         this_column as what_we_align_to,
+         another_column as something_short,  -- Is ok
+         case
+            -- This is aligned correctly with below
+            when indented then take_care
+            else try_harder
+         end as the_general_guidance
+      -- Even here we align with the line below
+      FROM my_table
+
+   .. note::
+
+      When fixing issues with comment indentation, SQLFluff
+      will attempt to keep comments in their original position
+      but if line length concerns make this difficult, it will
+      either abandon the fix, or move *same line* comments up and
+      *before* the line they are currently on. This is in line
+      with the assumption that comments on their own line refer
+      to the elements of code which they come *before*, not *after*.
+
+
+.. _hangingindents:
+
+Hanging Indents
+^^^^^^^^^^^^^^^
+
+One approach to indenting nested elements is a layout called a
+*hanging indent*. In this layout, there is no line break before the
+first nested element, but subsequent elements are indented to
+match the line position of that first element. Two examples might be:
+
+.. code-block:: sql
+
+   -- A select statement with two hanging indents:
+   SELECT no_line_break_before_me,
+          indented_to_match_the_first,
+          1 + (a
+               + b) AS another_more_complex_example
+   FROM my_table;
+
+   -- This TSQL example is also in essence a hanging indent:
+   DECLARE @prv_qtr_1st_dt DATETIME,
+           @last_qtr INT,
+           @last_qtr_first_mn INT,
+           @last_qtr_yr INT;
+
+In some circumstances this layout can be quite neat (the
+:code:`DECLARE` statement is a good example of this), however
+once indents are nested or indentation styles are mixed it
+can rapidly become confusing (as partially shown in the first
+example). Additionally, unless the leading element of the first
+line is very short, hanging indents use much *larger indents*
+than a traditional simple indent where a line break is used before
+the first element.
+
+Hanging indents have been supported in SQLFluff up to the 1.x
+versions, however **they will no longer by supported from 2.0.0**
+onwards. This is due to the ambiguity which they bring to
+fixing poorly formatted SQL. Take the following code:
+
+.. code-block:: sql
+
+   SELECT   this_is,
+   badly_formatted, code_and,
+      not_obvious,
+         what_was,
+   intended FROM my_table
+
+Given the lack of line break between :code:`SELECT` and
+:code:`this_is`, it would appear that the user is intending
+a hanging indent, however it is also plausible that they did
+not and they just forgot to add a line break between them.
+This ambiguity is unhelpful, both for SQLFluff as a tool,
+but also for people who write SQL that there two ways of
+indenting their SQL. Given SQLFluff aims to provide consistency
+in SQL layout and remove some of the burden of needing to make
+choices like this - and that it would be very unusual to keep
+*only hanging indents and disable traditional ones* - the only
+route left to consistency is to **not allow hanging indents**.
+Starting in 2.0.0, any hanging indents detected will be
+converted to traditional indents.
+
+.. _implicitindents:
+
+Implicit Indents
+^^^^^^^^^^^^^^^^
+
+A close cousin of the hanging indent is the *implicit indent*.
+While it does look a little like a hanging indent, it's much
+more consistent in its behaviour and is supported from SQLFluff
+2.0.0 onwards.
+
+An implicit indent is exactly like a normal indent, but doesn't
+have to be actually *taken* to influence the indentation of lines
+after it - it just needs to be left un-closed before the end of
+the line. These are normally available in clauses which take the
+form of :code:`KEYWORD <expression>`, like :code:`WHERE` clauses
+or :code:`CASE` expressions.
+
+.. code-block:: sql
+
+   -- This WHERE clause here takes advantage of an implicit indent.
+   SELECT *
+   FROM my_table
+   WHERE condition_a
+      AND condition_b;
+
+   -- With implicit indents disabled (which is currently the
+   -- default), the above formulation is not allowed, and instead
+   -- there should be a newline immediately after `WHERE` (which
+   -- is the location of the _implicit_ indent).
+   SELECT *
+   FROM my_table
+   WHERE
+      condition_a
+      AND condition_b;
+
+When addressing both indentation and line-length, implicit
+indents allow a slightly more compact layout, without significant
+drawbacks in legibility. They also enable a style much closer to
+some established style guides.
+
+They are however not recommended by many of the major style guides
+at time of writing (including the `dbt Labs SQL style guide`_
+and the `Mozilla SQL style guide`_), and so are disabled by default.
+To enable them, set the :code:`allow_implicit_indents` flag in
+:code:`sqluff.indentation` to :code:`True`.
+
+.. _templatedindents:
+
+Templated Indents
+^^^^^^^^^^^^^^^^^
+
+SQLFluff supports templated elements in code, such as those
+offered by jinja2 (or dbt which relies on it). For simple
+cases, templated elements are handled as you would expect
+by introducing additional indents into the layout.
+
+.. code-block:: SQL+Jinja
+
+   SELECT
+      a,
+      {% for n in ['b', 'c', 'd'] %}
+         -- This section is indented relative to 'a' because
+         -- it is inside a jinja for loop.
+         {{ n }},
+      {% endfor %}
+      e
+   FROM my_table
+
+This functionality can be turned off if you wish using the
+:code:`template_blocks_indent` option in your :ref:`config`.
+
+It's important to note here, that SQLFluff lints the code after
+it has been rendered, and so only has access to code which is
+still present after that process.
+
+.. code-block:: SQL+Jinja
+
+   SELECT
+      a,
+      {% if False %}
+      -- This section of the code cannot be linted because
+      -- it is never rendered due to the `if False` condition.
+      my    + poorly
+         +   spaced - and/indented AS    section_of_code
+      {% endif %}
+      e
+   FROM my_table
+
+More complex templated cases are usually characterised by templated
+tags *cutting across the parse tree*. This more formally is where the
+opening and closing tags of a templated section exist at different
+levels in the parsed structure. Starting in version 2.x, these will
+be treated differently (Prior to version 2.x, situations like this were sometimes
+handled inconsistently or incorrectly).
+
+Indentation should act as a visual cue to the structure of the
+written SQL, and as such, the most important thing is that template tags
+belonging to the same block structure use the same indentation.
+In the example below, this is the opening and closing elements of the
+second :code:`if` statement. If treated as a simple case, these tags
+would have different indents, because they are at different levels of
+the parse tree and so clearly there is a conflict to be resolved.
+
+The view SQLFluff takes on how to resolve this conflict is to pull
+all of the tags in this section down to the indent of the
+*least indented* (in the example below that would be the closing
+:code:`endif` tag). This is similar to the treatment of
+`C Preprocessor Directives`_, which are treated somewhat as being
+outside the structure of the rest of the file. In these cases,
+the content is also *not further indented* as in the simple case
+because it makes it harder to line up elements within the affected
+section and outside (in the example below the :code:`SELECT` and
+:code:`FROM` are a good illustration).
+
+.. code-block:: SQL+Jinja
+
+   SELECT
+      a,
+      {% if True %}
+         -- This is a simple case. The opening and closing tag are
+         -- both at the same level within the SELECT clause.
+         simple_case AS example,
+      {% endif %}
+      b,
+   {% if True %}
+      -- This is a complex case. The opening tag is within the SELECT
+      -- clause, but the closing tag is outside the statement
+      -- entirely.
+      complex_case AS example
+   FROM table_option_one
+   {% else %}
+      complex_case_two AS example
+   FROM table_option_two
+   {% endif %}
+
+
+.. _layoutconfig:
+
+Configuring Layout
+------------------
+
+Configuration for layout is spread across three places:
+
+#. Indent behavior for particular dialect elements is controlled by the parser.
+   This is because in the background SQLFluff inserts :code:`Indent`
+   and :code:`Dedent` tokens into the parse tree where those things
+   are expected. For more detail see :ref:`layoutindentconfig`.
+
+#. Configuration for the spacing and line position of particular
+   types of element (such as commas or operators) is set in the
+   :code:`layout` section of the config file. For more detail see
+   :ref:`layoutspacingconfig`.
+
+#. Some elements of layout are still controlled by rules directly.
+   These are usually very specific cases, see :ref:`ruleref` for
+   more details.
+
+
+.. _layoutindentconfig:
+
+Configuring indent locations
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+One of the key areas for this is the indentation of the
+:code:`JOIN` expression, which we'll use as an example.
+
+Semantically, a :code:`JOIN` expression is part of the :code:`FROM` expression
+and therefore would be expected to be indented. However according to many
+of the most common SQL style guides (including the `dbt Labs SQL style guide`_
+and the `Mozilla SQL style guide`_) the :code:`JOIN` keyword is expected to at
+the same indent as the :code:`FROM` keyword. By default, *SQLFluff* sides with
+the current consensus, which is to *not* indent the :code:`JOIN` keyword,
+however this is one element which is configurable.
+
+By setting values in the :code:`sqlfluff:indentation` section of your config
+file you can control how this is parsed.
+
+For example, the default indentation would be as follows:
+
+.. code-block:: sql
+
+   SELECT
+      a,
+      b
+   FROM my_table
+   JOIN another_table
+      ON
+         condition1
+         AND condition2
+
+By setting your config file to:
+
+.. code-block:: cfg
+
+   [sqlfluff:indentation]
+   indented_joins = True
+
+Then the expected indentation will be:
+
+.. code-block:: sql
+
+   SELECT
+      a,
+      b
+   FROM my_table
+      JOIN another_table
+         ON
+            condition1
+            AND condition2
+
+There is a similar :code:`indented_using_on` config (defaulted to :code:`True`)
+which can be set to :code:`False` to prevent the :code:`USING` or :code:`ON`
+clause from being indented, in which case the original SQL would become:
+
+.. code-block:: sql
+
+   SELECT
+      a,
+      b
+   FROM my_table
+   JOIN another_table
+   ON
+      condition1
+      AND condition2
+
+It's worth noting at this point, that for some users, the additional line
+break after :code:`ON` is unexpected, and this is a good example of an
+:ref:`implicit indent <implicitindents>`. By setting your config to:
+
+.. code-block:: cfg
+
+   [sqlfluff:indentation]
+   indented_using_on = False
+   allow_implicit_indents = True
+
+Then the expected indentation will be:
+
+.. code-block:: sql
+
+   SELECT
+      a,
+      b
+   FROM my_table
+   JOIN another_table
+   ON condition1
+      AND condition2
+
+There is also a similar :code:`indented_on_contents` config (defaulted to
+:code:`True`) which can be set to :code:`False` to align any :code:`AND`
+subsections of an :code:`ON` block with each other. If set to :code:`False`
+(assuming implicit indents are still enabled) the original SQL would become:
+
+.. code-block:: sql
+
+   SELECT
+      a,
+      b
+   FROM my_table
+   JOIN another_table
+      ON condition1
+      AND condition2
+
+These can also be combined, so if :code:`indented_using_on` config is set to
+:code:`False`, :code:`indented_on_contents` is also set to :code:`False`, and
+:code:`allow_implicit_indents` is set tot :code:`True` then the SQL would
+become:
+
+.. code-block:: sql
+
+   SELECT
+      a,
+      b
+   FROM my_table
+   JOIN another_table
+   ON condition1
+   AND condition2
+
+There is also a similar :code:`indented_ctes` config (defaulted to
+:code:`False`) which can be set to :code:`True` to enforce CTEs to be
+indented within the :code:`WITH` clause:
+
+.. code-block:: sql
+
+   WITH
+      some_cte AS (
+         SELECT 1 FROM table1
+      ),
+
+      some_other_cte AS (
+         SELECT 1 FROM table1
+      )
+
+   SELECT 1 FROM some_cte
+
+There is also a similar :code:`indented_then` config (defaulted to
+:code:`True`) which can be set to :code:`False` to allow :code:`THEN`
+without an indent after :code:`WHEN`:
+
+.. code-block:: sql
+
+   SELECT
+      a,
+      CASE
+         WHEN b >= 42 THEN
+            1
+         ELSE 0
+      END AS c
+   FROM some_table
+
+By default, *SQLFluff* aims to follow the most common approach
+to indentation. However, if you have other versions of indentation which are
+supported by published style guides, then please submit an issue on GitHub
+to have that variation supported by *SQLFluff*.
+
+.. _layoutspacingconfig:
+
+Configuring layout and spacing
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The :code:`[sqlfluff:layout]` section of the config controls the treatment of
+spacing and line breaks across all rules. The syntax of this section is very
+expressive; however in normal use, only very small alterations should be
+necessary from the :ref:`defaultconfig`.
+
+The syntax of the section headings here select by *type*, which corresponds
+to the :code:`type` defined in the dialect. For example the following section
+applies to elements of the *type* :code:`comma`, i.e. :code:`,`.
+
+.. code-block:: cfg
+
+   [sqlfluff:layout:type:comma]
+   spacing_before = touch
+   line_position = trailing
+
+Within these configurable sections there are a few key elements which are
+available:
+
+*  **Spacing Elements**: :code:`spacing_before`, :code:`spacing_after` and
+   :code:`spacing_within`. For each of these options, there are a few possible
+   settings:
+
+   *  The default spacing for all elements is :code:`single` unless otherwise
+      specified. In this state, elements will be spaced with a single space
+      character unless there is a line break between them.
+
+   *  The value of :code:`touch` allows line breaks, but if no line break is
+      present, then no space should be present. A great example of this is
+      the spacing before commas (as shown in the config above), where line
+      breaks may be allowed, but if not they should *touch* the element before.
+
+   *  Both of the above can be qualified with the :code:`:inline` modifier -
+      which prevents newlines within the segment. This is best illustrated
+      by the spacing found in a qualified identifier like
+      :code:`my_schema.my_table` which uses `touch:inline` or other clauses
+      where we want to force some elements to be on the same line.
+
+*  **Line Position**: set using the :code:`line_position` option. By default
+   this is unset, which implies no particular line position requirements. The
+   available options are:
+
+   *  :code:`trailing` and :code:`leading`, which are most common in the
+      placement of commas. Both of these settings *also* allow the option
+      of a comma on its own on a line, or in the middle of a line, *but*
+      if there is a line break on *either side* then they make sure it's
+      on the *correct side*. By default we assume *trailing* commas, but if
+      you (or your organisation) have settled on *leading* commas then
+      you should add the following section to your config:
+
+      .. code-block:: cfg
+
+         [sqlfluff:layout:type:comma]
+         line_position = leading
+
+   *  :code:`alone`, which means if there is a line break on either side,
+      then there must be a line break on *both sides* (i.e. that it should
+      be the only thing on that line.
+
+   *  All of the above options can be qualified with the :code:`:strict`
+      modifier - which prevents the *inline* case. For example:
+
+      .. code-block:: sql
+
+         -- Setting line_position to just `alone`
+         -- within [sqlfluff:layout:type:set_operator]
+         -- would not allow:
+         SELECT a
+         UNION SELECT b;
+         -- ...or...
+         SELECT a UNION
+         SELECT b;
+         -- but *would* allow both of the following:
+         SELECT a UNION SELECT b;
+         SELECT a
+         UNION
+         SELECT b;
+
+         -- However the default is set to `alone:strict`
+         -- then the *only* acceptable configuration is:
+         SELECT a
+         UNION
+         SELECT b;
+
+
+.. _`C Preprocessor Directives`: https://www.cprogramming.com/reference/preprocessor/
+.. _`dbt Labs SQL style guide`: https://github.com/dbt-labs/corp/blob/main/dbt_style_guide.md
+.. _`Mozilla SQL style guide`: https://docs.telemetry.mozilla.org/concepts/sql_style.html#joins

docs/source/configuration/rule_configuration.rst

@@ -0,0 +1,173 @@
+.. _ruleconfig:
+
+Rule Configuration
+------------------
+
+Rules can be configured with the :code:`.sqlfluff` config files.
+
+Common rule configurations can be set in the :code:`[sqlfluff:rules]` section.
+
+For example:
+
+.. code-block:: cfg
+
+   [sqlfluff:rules]
+   allow_scalar = True
+   single_table_references = consistent
+   unquoted_identifiers_policy = all
+
+Rule specific configurations are set in rule specific subsections.
+
+For example, enforce that keywords are upper case by configuring the rule
+:sqlfluff:ref:`CP01`:
+
+.. code-block:: cfg
+
+    [sqlfluff:rules:capitalisation.keywords]
+    # Keywords
+    capitalisation_policy = upper
+
+All possible options for rule sections are documented in :ref:`ruleref`.
+
+For an overview of the most common rule configurations that you may want to
+tweak, see :ref:`defaultconfig` (and use :ref:`ruleref` to find the
+available alternatives).
+
+.. _ruleselection:
+
+Enabling and Disabling Rules
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The decision as to which rules are applied to a given file is applied on a file
+by file basis, by the effective configuration for that file. There are two
+configuration values which you can use to set this:
+
+* :code:`rules`, which explicitly *enables* the specified rules. If this
+  parameter is unset or empty for a file, this implies "no selection" and
+  so "all rules" is taken to be the meaning.
+* :code:`exclude_rules`, which explicitly *disables* the specified rules.
+  This parameter is applied *after* the :code:`rules` parameter so can be
+  used to *subtract* from the otherwise enabled set.
+
+Each of these two configuration values accept a comma separated list of
+*references*. Each of those references can be:
+
+* a rule *code* e.g. :code:`LN01`
+* a rule *name* e.g. :code:`layout.indent`
+* a rule *alias*, which is often a deprecated *code* e.g. :code:`L003`
+* a rule *group* e.g. :code:`layout` or :code:`capitalisation`
+
+These different references can be mixed within a given expression, which
+results in a very powerful syntax for selecting exactly which rules are
+active for a given file.
+
+.. note::
+
+    It's worth mentioning here that the application of :code:`rules` and
+    :code:`exclude_rules`, with *groups*, *aliases* and *names*, in projects
+    with potentially multiple nested configuration files defining different
+    rules for different areas of a project can get very confusing very fast.
+    While this flexibility is intended for users to take advantage of, we do
+    have some recommendations about how to do this is a way that remains
+    manageable.
+
+    When considering configuration inheritance, each of :code:`rules` and
+    :code:`exclude_rules` will totally overwrite any values in parent config
+    files if they are set in a child file. While the subtraction operation
+    between both of them is calculated *"per file"*, there is no combination
+    operation between two definitions of :code:`rules` (just one overwrites
+    the other).
+
+    The effect of this is that we recommend one of two approaches:
+
+    #. Simply only use :code:`rules`. This has the upshot of each area of
+       your project being very explicit in which rules are enabled. When
+       that changes for part of your project you just reset the whole list
+       of applicable rules for that part of the project.
+    #. Set a single :code:`rules` value in your master project config file
+       and then only use :code:`exclude_rules` in sub-configuration files
+       to *turn off* specific rules for parts of the project where those
+       rules are inappropriate. This keeps the simplicity of only having
+       one value which is inherited, but allows slightly easier and simpler
+       rollout of new rules because we manage by exception.
+
+
+For example, to disable the rules :sqlfluff:ref:`LT08`
+and :sqlfluff:ref:`RF02`:
+
+.. code-block:: cfg
+
+    [sqlfluff]
+    exclude_rules = LT08, RF02
+
+To enable individual rules, configure :code:`rules`, respectively.
+
+For example, to enable :sqlfluff:ref:`RF02`:
+
+.. code-block:: cfg
+
+    [sqlfluff]
+    rules = RF02
+
+Rules can also be enabled/disabled by their grouping. Right now, the only
+rule grouping is :code:`core`. This will enable (or disable) a select group
+of rules that have been deemed 'core rules'.
+
+.. code-block:: cfg
+
+    [sqlfluff]
+    rules = core
+
+More information about 'core rules' can be found in the :ref:`ruleref`.
+
+Additionally, some rules have a special :code:`force_enable` configuration
+option, which allows to enable the given rule even for dialects where it is
+disabled by default. The rules that support this can be found in the
+:ref:`ruleref`.
+
+The default values can be seen in :ref:`defaultconfig`.
+
+See :ref:`ignoreconfig` for more information on how to turn ignore particular
+rules for specific lines, sections or files.
+
+Downgrading rules to warnings
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To keep displaying violations for specific rules, but not have those
+issues lead to a failed run, rules can be downgraded to *warnings*.
+Rules set as *warnings* won't cause a file to fail, but will still
+be shown in the CLI to warn users of their presence.
+
+The configuration of this behaves very like :code:`exclude_rules`
+above:
+
+.. code-block:: cfg
+
+    [sqlfluff]
+    warnings = LT01, LT04
+
+With this configuration, files with no other issues (other than
+those set to warn) will pass. If there are still other issues, then
+the file will still fail, but will show both warnings and failures.
+
+.. code-block::
+
+    == [test.sql] PASS
+    L:   2 | P:   9 | LT01 | WARNING: Missing whitespace before +
+    == [test2.sql] FAIL
+    L:   2 | P:   8 | CP02 | Unquoted identifiers must be consistently upper case.
+    L:   2 | P:  11 | LT01 | WARNING: Missing whitespace before +
+
+This is particularly useful as a transitional tool when considering
+the introduction on new rules on a project where you might want to
+make users aware of issues without blocking their workflow (yet).
+
+You can use either rule code or rule name for this setting.
+
+Layout & Spacing Configuration
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The :code:`[sqlfluff:layout]` section of the config controls the
+treatment of spacing and line breaks across all rules. To understand
+more about this section, see the section of the docs dedicated to
+layout: :ref:`layoutconfig`.

docs/source/configuration/setting_configuration.rst

@@ -0,0 +1,190 @@
+.. _setting_config:
+
+Setting Configuration
+=====================
+
+SQLFluff accepts configuration either through the command line or
+through configuration files. There is *rough* parity between the
+two approaches with the exception that *templating* configuration
+must be done via a file, because it otherwise gets slightly complicated.
+
+For details of what's available on the command line check out
+the :ref:`cliref`.
+
+.. _`config-files`:
+
+Configuration Files
+-------------------
+
+For file based configuration *SQLFluff* will look for the following
+files in order. Later files will (if found) will be used to overwrite
+any values read from earlier files.
+
+- :code:`setup.cfg`
+- :code:`tox.ini`
+- :code:`pep8.ini`
+- :code:`.sqlfluff`
+- :code:`pyproject.toml`
+
+Within these files, the first four will be read like a `cfg file`_, and
+*SQLFluff* will look for sections which start with :code:`sqlfluff`, and where
+subsections are delimited by a semicolon. For example the *jinjacontext*
+section will be indicated in the section started with
+:code:`[sqlfluff:jinjacontext]`.
+
+For example, a snippet from a :code:`.sqlfluff` file (as well as any of the
+supported cfg file types):
+
+.. code-block:: cfg
+
+    [sqlfluff]
+    templater = jinja
+    sql_file_exts = .sql,.sql.j2,.dml,.ddl
+
+    [sqlfluff:indentation]
+    indented_joins = False
+    indented_using_on = True
+    template_blocks_indent = False
+
+    [sqlfluff:templater]
+    unwrap_wrapped_queries = True
+
+    [sqlfluff:templater:jinja]
+    apply_dbt_builtins = True
+
+For the `pyproject.toml file`_, all valid sections start with
+:code:`tool.sqlfluff` and subsections are delimited by a dot. For example the
+*jinjacontext* section will be indicated in the section started with
+:code:`[tool.sqlfluff.jinjacontext]`.
+
+For example, a snippet from a :code:`pyproject.toml` file:
+
+.. code-block:: toml
+
+    [tool.sqlfluff.core]
+    templater = "jinja"
+    sql_file_exts = ".sql,.sql.j2,.dml,.ddl"
+
+    [tool.sqlfluff.indentation]
+    indented_joins = false
+    indented_using_on = true
+    template_blocks_indent = false
+
+    [tool.sqlfluff.templater]
+    unwrap_wrapped_queries = true
+
+    [tool.sqlfluff.templater.jinja]
+    apply_dbt_builtins = true
+
+    # For rule specific configuration, use dots between the names exactly
+    # as you would in .sqlfluff. In the background, SQLFluff will unpack the
+    # configuration paths accordingly.
+    [tool.sqlfluff.rules.capitalisation.keywords]
+    capitalisation_policy = "upper"
+
+.. _`cfg file`: https://docs.python.org/3/library/configparser.html
+.. _`pyproject.toml file`: https://www.python.org/dev/peps/pep-0518/
+
+
+.. _starter_config:
+
+New Project Configuration
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+When setting up a new project with SQLFluff, we recommend keeping your
+configuration file fairly minimal. The config file should act as a form
+of *documentation* for your team i.e. a record of what decisions you've
+made which govern how your format your SQL. By having a more concise
+config file, and only defining config settings where they differ from the
+defaults - you are more clearly stating to your team what choices you've made.
+
+*However*, there are also a few places where the *default* configuration
+is designed more for *existing projects*, rather than *fresh projects*, and
+so there is an opportunity to be a little stricter than you might otherwise
+be with an existing codebase.
+
+Here is a simple configuration file which would be suitable for a starter
+project:
+
+.. literalinclude:: /_partials/starter_config.cfg
+   :language: cfg
+
+
+.. _nesting:
+
+Nesting
+^^^^^^^
+
+**SQLFluff** uses **nesting** in its configuration files, with files
+closer *overriding* (or *patching*, if you will) values from other files.
+That means you'll end up with a final config which will be a patchwork
+of all the values from the config files loaded up to that path. The exception
+to this is the value for `templater`, which cannot be set in config files in
+subdirectories of the working directory.
+You don't **need** any config files to be present to make *SQLFluff*
+work. If you do want to override any values though SQLFluff will use
+files in the following locations in order, with values from later
+steps overriding those from earlier:
+
+0. *[...and this one doesn't really count]* There's a default config as
+   part of the SQLFluff package. You can find this below, in the
+   :ref:`defaultconfig` section.
+1. It will look in the user's os-specific app config directory.
+   On macOS and Unix this is `~/.config/sqlfluff`, Windows is
+   `<home>\\AppData\\Local\\sqlfluff\\sqlfluff`, for any of the filenames
+   above in the main :ref:`setting_config` section. If multiple are present, they will
+   *patch*/*override* each other in the order above.
+2. It will look for the same files in the user's home directory (~).
+3. *[if the current working directory is a subdirectory of the user's home directory (~)]*
+   It will look for the same files in all directories between the
+   user's home directory (~), and the current working directory.
+4. It will look for the same files in the current working directory.
+5. *[if parsing a file in a subdirectory of the current working directory]*
+   It will look for the same files in every subdirectory between the
+   current working dir and the file directory.
+6. It will look for the same files in the directory containing the file
+   being linted.
+
+This whole structure leads to efficient configuration, in particular
+in projects which utilise a lot of complicated templating.
+
+.. _in_file_config:
+
+In-File Configuration Directives
+--------------------------------
+
+In addition to configuration files mentioned above, SQLFluff also supports
+comment based configuration switching in files. This allows specific SQL
+file to modify a default configuration if they have specific needs.
+
+When used, these apply to the whole file, and are parsed from the file in
+an initial step before the rest of the file is properly parsed. This means
+they can be used for both rule configuration and also for parsing
+configuration.
+
+To use these, the syntax must start as an *inline sql comment* beginning
+with :code:`sqlfluff` (i.e. :code:`-- sqlfluff`). The line is then interpreted
+as a colon-separated address of the configuration value you wish to set.
+A few common examples are shown below:
+
+.. code-block:: sql
+
+    -- Set Indented Joins
+    -- sqlfluff:indentation:indented_joins:True
+
+    -- Set a smaller indent for this file
+    -- sqlfluff:indentation:tab_space_size:2
+
+    -- Set keywords to be capitalised
+    -- sqlfluff:rules:capitalisation.keywords:capitalisation_policy:upper
+
+    SELECT *
+    FROM a
+      JOIN b USING(c)
+
+We recommend only using this configuration approach for configuration that
+applies to one file in isolation. For configuration changes for areas of
+a project or for whole projects we recommend :ref:`nesting` of configuration
+files.
+
+This syntax is very similar to the method for :ref:`inline_ignoring_errors`.

docs/source/configuration/templating/dbt.rst

@@ -0,0 +1,146 @@
+.. _dbt_templater:
+
+:code:`dbt` templater
+^^^^^^^^^^^^^^^^^^^^^
+
+.. note::
+    From sqlfluff version 0.7.0 onwards, the dbt templater has been moved
+    to a separate plugin and python package. Projects that were already using
+    the dbt templater may initially fail after an upgrade to 0.7.0+. See the
+    installation instructions below to install the dbt templater.
+
+    dbt templating is still a relatively new feature added in 0.4.0 and
+    is still in very active development! If you encounter an issue, please
+    let us know in a GitHub issue or on the SQLFluff slack workspace.
+
+:code:`dbt` is not the default templater for *SQLFluff* (it is :code:`jinja`).
+:code:`dbt` is a complex tool, so using the default :code:`jinja` templater
+will be simpler. You should be aware when using the :code:`dbt` templater that
+you will be exposed to some of the complexity of :code:`dbt`. Users may wish to
+try both templaters and choose according to how they intend to use *SQLFluff*.
+
+A simple rule of thumb might be:
+
+- If you are using *SQLFluff* in a CI/CD context, where speed is not
+  critical but accuracy in rendering sql is, then the `dbt` templater
+  may be more appropriate.
+- If you are using *SQLFluff* in an IDE or on a git hook, where speed
+  of response may be more important, then the `jinja` templater may
+  be more appropriate.
+
+Pros:
+
+* Most (potentially all) macros will work
+
+Cons:
+
+* More complex, e.g. using it successfully may require deeper
+  understanding of your models and/or macros (including third-party macros)
+
+  * More configuration decisions to make
+  * Best practices are not yet established or documented
+
+* If your :code:`dbt` model files access a database at compile time, using
+  SQLFluff with the :code:`dbt` templater will **also** require access to a
+  database.
+
+  * Note that you can often point SQLFluff and the :code:`dbt` templater at a
+    test database (i.e. it doesn't have to be the production database).
+
+* Runs slower
+
+Installation & Configuration
+""""""""""""""""""""""""""""
+
+In order to get started using *SQLFluff* with a dbt project you will
+first need to install the relevant `dbt adapter`_ for your dialect
+and the :code:`sqlfluff-templater-dbt` package using
+your package manager of choice (e.g.
+:code:`pip install dbt-postgres sqlfluff-templater-dbt`) and then will need the
+following configuration:
+
+.. _`dbt adapter`: https://docs.getdbt.com/docs/available-adapters
+
+In *.sqlfluff*:
+
+.. code-block:: cfg
+
+    [sqlfluff]
+    templater = dbt
+
+In *.sqlfluffignore*:
+
+.. code-block:: text
+
+    target/
+    # dbt <1.0.0
+    dbt_modules/
+    # dbt >=1.0.0
+    dbt_packages/
+    macros/
+
+You can set the dbt project directory, profiles directory and profile with:
+
+.. code-block:: cfg
+
+    [sqlfluff:templater:dbt]
+    project_dir = <relative or absolute path to dbt_project directory>
+    profiles_dir = <relative or absolute path to the directory that contains the profiles.yml file>
+    profile = <dbt profile>
+    target = <dbt target>
+
+.. note::
+
+    If the `profiles_dir` setting is omitted, SQLFluff will look for the profile
+    in the default location, which varies by operating system. On Unix-like
+    operating systems (e.g. Linux or macOS), the default profile directory is
+    `~/.dbt/`. On Windows, you can determine your default profile directory by
+    running `dbt debug --config-dir`.
+
+To use builtin dbt Jinja functions SQLFluff provides a configuration option
+that enables usage within templates.
+
+.. code-block:: cfg
+
+    [sqlfluff:templater:jinja]
+    apply_dbt_builtins = True
+
+This will provide dbt macros like `ref`, `var`, `is_incremental()`. If the need
+arises builtin dbt macros can be customised via Jinja macros in `.sqlfluff`
+configuration file.
+
+.. code-block:: cfg
+
+    [sqlfluff:templater:jinja:macros]
+    # Macros provided as builtins for dbt projects
+    dbt_ref = {% macro ref(model_ref) %}{{model_ref}}{% endmacro %}
+    dbt_source = {% macro source(source_name, table) %}{{source_name}}_{{table}}{% endmacro %}
+    dbt_config = {% macro config() %}{% for k in kwargs %}{% endfor %}{% endmacro %}
+    dbt_var = {% macro var(variable, default='') %}item{% endmacro %}
+    dbt_is_incremental = {% macro is_incremental() %}True{% endmacro %}
+
+If your project requires that you pass variables to dbt through command line,
+you can specify them in `template:dbt:context` section of `.sqlfluff`.
+See below configuration and its equivalent dbt command:
+
+.. code-block:: cfg
+
+    [sqlfluff:templater:dbt:context]
+    my_variable = 1
+
+.. code-block:: text
+
+    dbt run --vars '{"my_variable": 1}'
+
+Known Caveats
+"""""""""""""
+
+- To use the dbt templater, you must set `templater = dbt` in the `.sqlfluff`
+  config file in the directory where sqlfluff is run. The templater cannot
+  be changed in `.sqlfluff` files in subdirectories.
+- In SQLFluff 0.4.0 using the dbt templater requires that all files
+  within the root and child directories of the dbt project must be part
+  of the project. If there are deployment scripts which refer to SQL files
+  not part of the project for instance, this will result in an error.
+  You can overcome this by adding any non-dbt project SQL files to
+  .sqlfluffignore.

docs/source/configuration/templating/index.rst

@@ -0,0 +1,100 @@
+.. _templateconfig:
+
+Templating Configuration
+------------------------
+
+This section explains how to configure templating for SQL files.
+
+When writing SQL files, users might utilise some kind of templating.
+The SQL file itself is written with placeholders which get rendered to proper
+SQL at run time.
+This can range from very simple placeholder templating to complex Jinja
+templating.
+
+SQLFluff supports templated sections in SQL, see :ref:`templater`.
+This is achieved by the following set of operations:
+
+1. SQLFluff pre-renders the templated SQL
+2. SQLFluff applies the lint and fix operations to the rendered file
+3. SQLFluff backports the rule violations to the templated section of the SQL.
+
+SQLFluff does not automatically have access to the same environment used in
+production template setup. This means it is necessary to either provide that
+environment or provide dummy values to effectively render the template and
+generate valid SQL. Refer to the templater sections below for details.
+
+SQLFluff natively supports the following templating engines
+
+- :ref:`jinja_templater`
+- :ref:`placeholder_templater`
+- :ref:`python_templater`
+
+Also, SQLFluff has an integration to use :code:`dbt` as a templater.
+
+- :ref:`dbt_templater` (via plugin which is covered in a different section).
+
+.. note::
+
+    Templaters may not be able to generate a rendered SQL that cover
+    the entire raw file.
+
+    For example, if the raw SQL uses a :code:`{% if condition %}` block,
+    the rendered version of the template will only include either the
+    :code:`{% then %}` or the :code:`{% else %}` block (depending on the
+    provided configuration for the templater), but not both.
+
+    In this case, because SQLFluff linting can only operate on the output
+    of the templater, some areas of the raw SQL will never be seen by the
+    linter and will not be covered by lint rules.
+
+    This is functionality we hope to support in future.
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Templater Specific Configuration:
+
+   jinja
+   placeholder
+   python
+   dbt
+
+.. _generic_variable_templating:
+
+Generic Variable Templating
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Variables are available in all the templaters.
+By default the templating engine will expect variables for templating to be
+available in the config, and the templater will be look in the section
+corresponding to the context for that templater. By convention, the config for
+the ``jinja`` templater is found in the ``sqlfluff:templater:jinja:context``
+section, the config for the ``python`` templater is found in the
+``sqlfluff:templater:python:context`` section, the one for the ``placeholder``
+templater is found in the ``sqlfluff:templater:placeholder:context`` section.
+
+For example, if passed the following *.sql* file:
+
+.. code-block:: SQL+Jinja
+
+    SELECT {{ num_things }} FROM {{ tbl_name }} WHERE id > 10 LIMIT 5
+
+...and the following configuration in *.sqlfluff* in the same directory:
+
+.. code-block:: cfg
+
+    [sqlfluff:templater:jinja:context]
+    num_things=456
+    tbl_name=my_table
+
+...then before parsing, the sql will be transformed to:
+
+.. code-block:: sql
+
+    SELECT 456 FROM my_table WHERE id > 10 LIMIT 5
+
+.. note::
+
+    If there are variables in the template which cannot be found in
+    the current configuration context, then this will raise a
+    `SQLTemplatingError` and this will appear as a violation without
+    a line number, quoting the name of the variable that couldn't be found.

docs/source/configuration/templating/jinja.rst

@@ -0,0 +1,417 @@
+.. _jinja_templater:
+
+Jinja templater
+^^^^^^^^^^^^^^^
+
+The Jinja templater uses Jinja2_ to render templates.
+
+.. _Jinja2: https://jinja.palletsprojects.com/
+
+There are multiple, complementary ways of configuring the Jinja templater.
+
+- Reading variables and Jinja macros directly from the SQLFLuff config file
+- Loading macros from a path
+- Using a library
+
+.. list-table:: Overview of Jinja templater's configuration options
+   :header-rows: 1
+
+   * - Configuration
+     - Variables
+     - Macros
+     - Filters
+     - Documentation
+   * - Config file
+     - ✅
+     - ✅
+     - ❌
+     - `Complex Jinja Variable Templating`_ and `Jinja Macro Templating (from config)`_
+   * - Macro Path
+     - ❌
+     - ✅
+     - ❌
+     - `Jinja Macro Templating (from file)`_
+   * - Library
+     - ✅
+     - ✅
+     - ✅
+     - `Library Templating`_
+
+For example, a snippet from a :code:`.sqlfluff` file that uses all config
+options:
+
+.. code-block:: cfg
+
+    [sqlfluff]
+    templater = jinja
+
+    [sqlfluff:templater:jinja]
+    apply_dbt_builtins = True
+    load_macros_from_path = my_macros
+    loader_search_path = included_templates
+    library_path = sqlfluff_libs
+    exclude_macros_from_path = my_macros_exclude
+
+    [sqlfluff:templater:jinja:context]
+    my_list = ['a', 'b', 'c']
+    MY_LIST = ("d", "e", "f")
+    my_where_dict = {"field_1": 1, "field_2": 2}
+
+    [sqlfluff:templater:jinja:macros]
+    a_macro_def = {% macro my_macro(n) %}{{ n }} + {{ n * 2 }}{% endmacro %}
+
+Complex Jinja Variable Templating
+"""""""""""""""""""""""""""""""""
+
+Apart from the Generic variable templating that is supported for all
+templaters, two more advanced features of variable templating are available for
+Jinja.
+
+*case sensitivity* and *native python types*.
+Both are illustrated in the following example:
+
+.. code-block:: cfg
+
+    [sqlfluff:templater:jinja:context]
+    my_list = ['a', 'b', 'c']
+    MY_LIST = ("d", "e", "f")
+    my_where_dict = {"field_1": 1, "field_2": 2}
+
+.. code-block:: SQL+Jinja
+
+    SELECT
+        {% for elem in MY_LIST %}
+            '{{elem}}' {% if not loop.last %}||{% endif %}
+        {% endfor %} as concatenated_list
+    FROM tbl
+    WHERE
+        {% for field, value in my_where_dict.items() %}
+            {{field}} = {{value}} {% if not loop.last %}and{% endif %}
+        {% endfor %}
+
+...will render as...
+
+.. code-block:: sql
+
+    SELECT
+        'd' || 'e' || 'f' as concatenated_list
+    FROM tbl
+    WHERE
+        field_1 = 1 and field_2 = 2
+
+Note that the variable was replaced in a case sensitive way and that the
+settings in the config file were interpreted as native python types.
+
+Jinja Macro Templating (from config)
+""""""""""""""""""""""""""""""""""""
+
+Macros (which also look and feel like *functions* are available only in the
+*jinja* templater. Similar to :ref:`generic_variable_templating`, these are
+specified in config files, what's different in this case is how they are named.
+Similar to the *context* section above, macros are configured separately in the
+*macros* section of the config.
+Consider the following example.
+
+If passed the following *.sql* file:
+
+.. code-block:: SQL+Jinja
+
+    SELECT {{ my_macro(6) }} FROM some_table
+
+...and the following configuration in *.sqlfluff* in the same directory (note
+the tight control of whitespace):
+
+.. code-block:: cfg
+
+    [sqlfluff:templater:jinja:macros]
+    a_macro_def = {% macro my_macro(n) %}{{ n }} + {{ n * 2 }}{% endmacro %}
+
+...then before parsing, the sql will be transformed to:
+
+.. code-block:: sql
+
+    SELECT 6 + 12 FROM some_table
+
+Note that in the code block above, the variable name in the config is
+*a_macro_def*, and this isn't apparently otherwise used anywhere else.
+Broadly this is accurate, however within the configuration loader this will
+still be used to overwrite previous *values* in other config files. As such
+this introduces the idea of config *blocks* which could be selectively
+overwritten by other configuration files downstream as required.
+
+Jinja Macro Templating (from file)
+""""""""""""""""""""""""""""""""""
+
+In addition to macros specified in the config file, macros can also be
+loaded from files or folders. This is specified in the config file:
+
+.. code-block:: cfg
+
+    [sqlfluff:templater:jinja]
+    load_macros_from_path = my_macros,other_macros
+
+``load_macros_from_path`` is a comma-separated list of :code:`.sql` files or
+folders. Locations are *relative to the config file*. For example, if the
+config file above was found at :code:`/home/my_project/.sqlfluff`, then
+SQLFluff will look for macros in the folders :code:`/home/my_project/my_macros/`
+and  :code:`/home/my_project/other_macros/`, including any of their subfolders.
+Any macros defined in the config will always take precedence over a macro
+defined in the path.
+
+``exclude_macros_from_path`` works in the same manner as ``load_macros_from_path`` but
+allows you to have sqlfluff ignore certain macros. This can be useful if you have
+custom jinja tags.
+
+Macros loaded from these files are available in every :code:`.sql` file without
+requiring a Jinja :code:`include` or :code:`import`.  They are loaded into the
+`Jinja Global Namespace <https://jinja.palletsprojects.com/en/3.1.x/api/#global-namespace>`_.
+
+**Note:** The :code:`load_macros_from_path` setting also defines the search
+path for Jinja
+`include <https://jinja.palletsprojects.com/en/3.1.x/templates/#include>`_ or
+`import <https://jinja.palletsprojects.com/en/3.1.x/templates/#import>`_.
+As with loaded macros, subdirectories are also supported. For example,
+if :code:`load_macros_from_path` is set to :code:`my_macros`, and there is a
+file :code:`my_macros/subdir/my_file.sql`, you can do:
+
+.. code-block:: jinja
+
+   {% include 'subdir/my_file.sql' %}
+
+If you would like to define the Jinja search path without also loading the
+macros into the global namespace, use the :code:`loader_search_path` setting
+instead.
+
+.. note::
+
+    Throughout the templating process **whitespace** will still be treated
+    rigorously, and this includes **newlines**. In particular you may choose
+    to provide *dummy* macros in your configuration different from the actual
+    macros used in production.
+
+    **REMEMBER:** The reason SQLFluff supports macros is to *enable* it to parse
+    templated sql without it being a blocker. It shouldn't be a requirement that
+    the *templating* is accurate - it only needs to work well enough that
+    *parsing* and *linting* are helpful.
+
+.. _builtin_jinja_blocks:
+
+Builtin Jinja Macro Blocks
+""""""""""""""""""""""""""
+
+One of the main use cases which inspired *SQLFluff* as a project was `dbt`_.
+It uses jinja templating extensively and leads to some users maintaining large
+repositories of sql files which could potentially benefit from some linting.
+
+.. note::
+    *SQLFluff* has now a tighter integration with dbt through the "dbt" templater.
+    It is the recommended templater for dbt projects. If used, it eliminates the
+    need for the overrides described in this section.
+
+    To use the dbt templater, go to :ref:`dbt_templater`.
+
+*SQLFluff* anticipates this use case and provides some built in macro blocks
+in the :ref:`defaultconfig` which assist in getting started with `dbt`_
+projects. In particular it provides mock objects for:
+
+* *ref*: The mock version of this provided simply returns the model reference
+  as the name of the table. In most cases this is sufficient.
+* *config*: A regularly used macro in `dbt`_ to set configuration values. For
+  linting purposes, this makes no difference and so the provided macro simply
+  returns nothing.
+
+.. note::
+    If there are other builtin macros which would make your life easier,
+    consider submitting the idea (or even better a pull request) on `github`_.
+
+.. _`dbt`: https://www.getdbt.com/
+.. _`github`: https://www.github.com/sqlfluff/sqlfluff
+
+.. _jinja_library_templating:
+
+Library Templating
+""""""""""""""""""
+
+If using *SQLFluff* with jinja as your templater, you may have library
+function calls within your sql files that can not be templated via the
+normal macro templating mechanisms:
+
+.. code-block:: SQL+Jinja
+
+    SELECT foo, bar FROM baz {{ dbt_utils.group_by(2) }}
+
+To template these libraries, you can use the `sqlfluff:jinja:library_path`
+config option:
+
+.. code-block:: cfg
+
+    [sqlfluff:templater:jinja]
+    library_path = sqlfluff_libs
+
+This will pull in any python modules from that directory and allow sqlfluff
+to use them in templates. In the above example, you might define a file at
+`sqlfluff_libs/dbt_utils.py` as:
+
+.. code-block:: python
+
+    def group_by(n):
+        return "GROUP BY 1,2"
+
+
+If an `__init__.py` is detected, it will be loaded alongside any modules and
+submodules found within the library path.
+
+.. code-block:: SQL+Jinja
+
+   SELECT
+      {{ custom_sum('foo', 'bar') }},
+      {{ foo.bar.another_sum('foo', 'bar') }}
+   FROM
+      baz
+
+`sqlfluff_libs/__init__.py`:
+
+.. code-block:: python
+
+    def custom_sum(a: str, b: str) -> str:
+        return a + b
+
+`sqlfluff_libs/foo/__init__.py`:
+
+.. code-block:: python
+
+    # empty file
+
+`sqlfluff_libs/foo/bar.py`:
+
+.. code-block:: python
+
+     def another_sum(a: str, b: str) -> str:
+        return a + b
+
+Additionally, the library can be used to expose `Jinja Filters <https://jinja.palletsprojects.com/en/3.1.x/templates/#filters>`_
+to the Jinja environment used by SQLFluff.
+
+This is achieve by setting a global variable named ``SQLFLUFF_JINJA_FILTERS``.
+``SQLFLUFF_JINJA_FILTERS`` is a dictionary where
+
+* dictionary keys map to the Jinja filter name
+* dictionary values map to the Python callable
+
+For example, to make the Airflow filter ``ds`` available to SQLFLuff, add
+the following to the `__init__.py` of the library:
+
+.. code-block:: python
+
+     # https://github.com/apache/airflow/blob/main/airflow/templates.py#L53
+     def ds_filter(value: datetime.date | datetime.time | None) -> str | None:
+        """Date filter."""
+        if value is None:
+            return None
+        return value.strftime("%Y-%m-%d")
+
+     SQLFLUFF_JINJA_FILTERS = {"ds": ds_filter}
+
+Now, ``ds`` can be used in SQL
+
+.. code-block:: SQL+Jinja
+
+    SELECT "{{ "2000-01-01" | ds }}";
+
+Jinja loader search path
+""""""""""""""""""""""""
+
+The Jinja environment can be configured to search for files included with
+`include <https://jinja.palletsprojects.com/en/3.1.x/templates/#include>`_ or
+`import <https://jinja.palletsprojects.com/en/3.1.x/templates/#import>`_ in a
+list of folders. This is specified in the config file:
+
+.. code-block:: cfg
+
+    [sqlfluff:templater:jinja]
+    loader_search_path = included_templates,other_templates
+
+``loader_search_path`` is a comma-separated list of folders. Locations are
+*relative to the config file*. For example, if the config file above was found
+at :code:`/home/my_project/.sqlfluff`, then SQLFluff will look for included
+files in the folders :code:`/home/my_project/included_templates/` and
+:code:`/home/my_project/other_templates/`, including any of their subfolders.
+For example, this will read from
+:code:`/home/my_project/included_templates/my_template.sql`:
+
+.. code-block:: jinja
+
+   {% include 'included_templates/my_template.sql' %}
+
+Any folders specified in the :code:`load_macros_from_path` setting are
+automatically appended to the ``loader_search_path``.  It is not necessary to
+specify a given directory in both settings.
+
+Unlike the :code:`load_macros_from_path` setting, any macros within these
+folders are *not* automatically loaded into the global namespace.  They must be
+explicitly imported using the
+`import <https://jinja.palletsprojects.com/en/3.1.x/templates/#import>`_ Jinja
+directive.  If you would like macros to be automatically included in the
+global Jinja namespace, use the :code:`load_macros_from_path` setting instead.
+
+Interaction with ``--ignore=templating``
+""""""""""""""""""""""""""""""""""""""""
+
+Ignoring Jinja templating errors provides a way for users to use SQLFluff
+while reducing or avoiding the need to spend a lot of time adding variables
+to ``[sqlfluff:templater:jinja:context]``.
+
+When ``--ignore=templating`` is enabled, the Jinja templater behaves a bit
+differently. This additional behavior is *usually* but not *always* helpful
+for making the file at least partially parsable and fixable. It definitely
+doesn’t **guarantee** that every file can be fixed, but it’s proven useful for
+some users.
+
+Here's how it works:
+
+* Within the expanded SQL, undefined variables are automatically *replaced*
+  with the corresponding string value.
+* If you do: ``{% include query %}``, and the variable ``query`` is not
+  defined, it returns a “file” containing the string ``query``.
+* If you do: ``{% include "query_file.sql" %}``, and that file does not exist
+  or you haven’t configured a setting for ``load_macros_from_path`` or
+  ``loader_search_path``, it returns a “file” containing the text
+  ``query_file``.
+
+For example:
+
+.. code-block:: SQL+Jinja
+
+   select {{ my_variable }}
+   from {% include "my_table.sql" %}
+
+is interpreted as:
+
+.. code-block:: sql
+
+   select my_variable
+   from my_table
+
+The values provided by the Jinja templater act *a bit* (not exactly) like a
+mixture of several types:
+
+* ``str``
+* ``int``
+* ``list``
+* Jinja's ``Undefined`` `class <https://jinja.palletsprojects.com/en/3.1.x/api/#jinja2.Undefined>`_
+
+Because the values behave like ``Undefined``, it's possible to replace them
+using Jinja's ``default()`` `filter <https://jinja.palletsprojects.com/en/3.1.x/templates/#jinja-filters.default>`_.
+For example:
+
+.. code-block:: SQL+Jinja
+
+      select {{ my_variable | default("col_a") }}
+      from my_table
+
+is interpreted as:
+
+.. code-block:: sql
+
+      select col_a
+      from my_table

docs/source/configuration/templating/placeholder.rst

@@ -0,0 +1,106 @@
+.. _placeholder_templater:
+
+Placeholder templater
+^^^^^^^^^^^^^^^^^^^^^
+
+Libraries such as SQLAlchemy or Psycopg use different parameter placeholder
+styles to mark where a parameter has to be inserted in the query.
+
+For example a query in SQLAlchemy can look like this:
+
+.. code-block:: sql
+
+    SELECT * FROM table WHERE id = :myid
+
+At runtime `:myid` will be replace by a value provided by the application and
+escaped as needed, but this is not standard SQL and cannot be parsed as is.
+
+In order to parse these queries is then necessary to replace these
+placeholders with sample values, and this is done with the placeholder
+templater.
+
+Placeholder templating can be enabled in the config using:
+
+.. code-block:: cfg
+
+    [sqlfluff]
+    templater = placeholder
+
+A few common styles are supported:
+
+.. code-block:: sql
+   :force:
+
+    -- colon
+    WHERE bla = :my_name
+
+    -- colon_nospaces
+    -- (use with caution as more prone to false positives)
+    WHERE bla = table:my_name
+
+    -- colon_optional_quotes
+    SELECT :"column" FROM :table WHERE bla = :'my_name'
+
+    -- numeric_colon
+    WHERE bla = :2
+
+    -- pyformat
+    WHERE bla = %(my_name)s
+
+    -- dollar
+    WHERE bla = $my_name or WHERE bla = ${my_name}
+
+    -- question_mark
+    WHERE bla = ?
+
+    -- numeric_dollar
+    WHERE bla = $3 or WHERE bla = ${3}
+
+    -- percent
+    WHERE bla = %s
+
+    -- ampersand
+    WHERE bla = &s or WHERE bla = &{s} or USE DATABASE MARK_{ENV}
+
+These can be configured by setting `param_style` to the names above:
+
+.. code-block:: cfg
+
+    [sqlfluff:templater:placeholder]
+    param_style = colon
+    my_name = 'john'
+
+then you can set sample values for each parameter, like `my_name`
+above. Notice that the value needs to be escaped as it will be replaced as a
+string during parsing. When the sample values aren't provided, the templater
+will use parameter names themselves by default.
+
+When parameters are positional, like `question_mark`, then their name is
+simply the order in which they appear, starting with `1`.
+
+.. code-block:: cfg
+
+    [sqlfluff:templater:placeholder]
+    param_style = question_mark
+    1 = 'john'
+
+In case you need a parameter style different from the ones above, you can pass
+a custom regex.
+
+.. code-block:: cfg
+
+    [sqlfluff:templater:placeholder]
+    param_regex = __(?P<param_name>[\w_]+)__
+    my_name = 'john'
+
+N.B. quotes around `param_regex` in the config are
+interpreted literally by the templater.
+e.g. `param_regex='__(?P<param_name>[\w_]+)__'` matches
+`'__some_param__'` not `__some_param__`
+
+the named parameter `param_name` will be used as the key to replace, if
+missing, the parameter is assumed to be positional and numbers are used
+instead.
+
+Also consider making a pull request to the project to have your style added,
+it may be useful to other people and simplify your configuration.

docs/source/configuration/templating/python.rst

@@ -0,0 +1,58 @@
+.. _python_templater:
+
+Python templater
+^^^^^^^^^^^^^^^^
+
+Uses native Python f-strings. As described in
+:ref:`generic_variable_templating`, an example usage would look be
+configured as follows:
+
+If passed the following *.sql* file:
+
+.. code-block::
+
+    SELECT * FROM {tbl_name}
+
+...and the following configuration in *.sqlfluff* in the same directory:
+
+.. code-block:: cfg
+
+    [sqlfluff]
+    templater = python
+
+    [sqlfluff:templater:python:context]
+    tbl_name = my_table
+
+...then before parsing, the sql will be transformed to:
+
+.. code-block:: sql
+
+    SELECT * FROM my_table
+
+
+Complex Python Variable Templating
+""""""""""""""""""""""""""""""""""""
+
+`Python string formatting`_ supports accessing object attributes
+via dot notation (e.g. :code:`{foo.bar}`).  However, since we cannot create Python
+objects within configuration files, we need a workaround in order to provide
+dummy values to render templates containing these values.  The SQLFluff
+python templater will interpret any variable containing a "." as a
+dictionary lookup on the *magic* fixed context key :code:`sqlfluff`.
+
+.. code-block::
+
+    -- this SQL
+    SELECT * FROM {foo.bar}
+
+    -- becomes this
+    SELECT * FROM {sqlfluff["foo.bar"]}
+
+..which can be populated using the following configuration:
+
+.. code-block:: cfg
+
+    [sqlfluff:templater:python:context]
+    sqlfluff = {"foo.bar": "abc"}
+
+.. _`Python string formatting`: https://docs.python.org/3/library/string.html#format-string-syntax

docs/source/developingplugins.rst

@@ -1,61 +0,0 @@
-.. _developingpluginsref:
-
-Developing Plugins
-==================
-
-*SQLFluff* is extensible through "plugins". We use the `pluggy library`_
-to make linting Rules pluggable, which enable users to implement rules that
-are just too "organization specific" to be shared, or too platform specific
-to be included in the core library.
-
-.. _`pluggy library`: https://pluggy.readthedocs.io/en/latest/
-
-Creating a plugin
------------------
-
-We have an example plugin in
-`sqlfluff/plugins/sqlfluff-plugin-example`_ which you can use as a template
-for rules, or the `sqlfluff/plugins/sqlfluff-templater-dbt`_ which you can
-use as a template for templater plugins.
-
-Few things to note about plugins:
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Currently, only Rules and Templaters can be added through plugins. Over time
-we expect more elements of SQLFluff will be extensible with plugins. Each
-plugin can implement multiple Rules or Templaters.
-
-We recomment that the name of a plugin should start with *"sqlfluff-"* to be
-clear on the purpose of your plugin.
-
-A plugin may need to include a default configuration if its rules
-are configurable: use plugin default configurations **only for that reason**!
-We advise against overwriting core configurations by using a default
-plugin configuration, as there is no mechanism in place to enforce precendence
-between the core library configs and plugin configs,
-and multiple plugins could clash.
-
-A plugin Rule class name should have the structure:
-"Rule_PluginName_L000". The 'L' can be any letter
-and is meant to categorize rules; you could use the
-letter 'S' to denote rules that enforce security checks
-for example.
-
-A plugin Rule code includes the PluginName,
-so a rule "Rule_L000" in core will have code "L000",
-while "Rule_PluginName_L000" will have code "PluginName_L000".
-Codes are used to display errors, they are also used as configuration keys.
-
-We make it easy for plugin developers to test their rules by
-exposing a testing library in *sqlfluff.testing*.
-
-.. _`sqlfluff/plugins/sqlfluff-plugin-example`: https://github.com/sqlfluff/sqlfluff/tree/main/plugins/sqlfluff-plugin-example
-.. _`sqlfluff/plugins/sqlfluff-templater-dbt`: https://github.com/sqlfluff/sqlfluff/tree/main/plugins/sqlfluff-templater-dbt
-
-Giving feedback
----------------
-
-Would you like to have other parts of *SQLFluff* be "pluggable"?
-Tell us about it in a `GitHub issue`_ 😄.
-
-.. _`GitHub issue`: https://github.com/sqlfluff/sqlfluff/issues/new?assignees=&labels=enhancement&template=enhancement.md

docs/source/developingrules.rst

@@ -1,52 +0,0 @@
-.. _developingrulesref:
-
-Developing Rules
-================
-
-`Rules` in `SQLFluff` are implemented as `crawlers`. These are entities
-which work their way through the parsed structure of a query to evaluate
-a particular rule or set of rules. The intent is that the definition of
-each specific rule should be really streamlined and only contain the logic
-for the rule itself, with all the other mechanics abstracted away.
-
-Base Rules
-----------
-
-`base_rules` Module
-^^^^^^^^^^^^^^^^^^^
-
-.. automodule:: sqlfluff.core.rules.base
-   :members:
-
-Functional API
---------------
-These newer modules provide a higher-level API for rules working with segments
-and slices. Rules that need to navigate or search the parse tree may benefit
-from using these. Eventually, the plan is for **all** rules to use these
-modules. As of December 30, 2021, 17+ rules use these modules.
-
-The modules listed below are submodules of `sqlfluff.core.rules.functional`.
-
-`segments` Module
-^^^^^^^^^^^^^^^^^
-
-.. automodule:: sqlfluff.core.rules.functional.segments
-   :members:
-
-`segment_predicates` Module
-^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-.. automodule:: sqlfluff.core.rules.functional.segment_predicates
-   :members:
-
-`raw_file_slices` Module
-^^^^^^^^^^^^^^^^^^^^^^^^
-
-.. automodule:: sqlfluff.core.rules.functional.raw_file_slices
-   :members:
-
-`raw_file_slice_predicates` Module
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-.. automodule:: sqlfluff.core.rules.functional.raw_file_slice_predicates
-   :members:

docs/source/dialects.rst

@@ -1,174 +0,0 @@
-.. _dialectref:
-
-Dialects Reference
-==================
-
-SQLFluff is designed to be flexible in supporting a variety of dialects.
-Not all potential dialects are supported so far, but several have been
-implemented by the community. Below are a list of the currently available
-dialects. Each inherits from another, up to the root `ansi` dialect.
-
-For a canonical list of supported dialects, run the
-:program:`sqlfluff dialects` command, which will output a list of the
-current dialects available on your installation of SQLFluff.
-
-.. note::
-
-    For technical users looking to add new dialects or add new features
-    to existing ones, the dependent nature of how dialects have been
-    implemented is to try and reduce the amount of repetition in how
-    different elements are defined. As an example, when we say that
-    the :ref:`redshift_dialect_ref` dialect *inherits* from the
-    :ref:`postgres_dialect_ref` dialect this is not because there
-    is an agreement between those projects which means that features
-    in one must end up in the other, but that the design of the
-    :ref:`redshift_dialect_ref` dialect was heavily *inspired* by the
-    postgres dialect and therefore when defining the dialect within
-    sqlfuff it makes sense to use :ref:`postgres_dialect_ref` as a
-    starting point rather than starting from scratch.
-
-    Consider when adding new features to a dialect:
-
-    - Should I be adding it just to this dialect, or adding it to
-      a *parent* dialect?
-    - If I'm creating a new dialect, which dialect would be best to
-      inherit from?
-    - Will the feature I'm adding break any *downstream* dependencies
-      within dialects which inherit from this one?
-
-.. _ansi_dialect_ref:
-
-ANSI
-----
-
-This is the base dialect which holds most of the definitions of common
-SQL commands and structures. If the dialect which you're actually using
-isn't specifically implemented by SQLFluff, using this dialect is a good
-place to start.
-
-This dialect doesn't intend to be brutal in adhering to (and only to) the
-ANSI SQL spec *(mostly because ANSI charges for access to that spec)*. It aims
-to be a representation of vanilla SQL before any other project adds their
-spin to it, and so may contain a slightly wider set of functions than actually
-available in true ANSI SQL.
-
-.. _bigquery_dialect_ref:
-
-BigQuery
---------
-
-The dialect for `Google BigQuery`_.
-
-.. _`Google BigQuery`: https://cloud.google.com/bigquery/
-
-.. _exasol_dialect_ref:
-
-Exasol
-------
-
-The dialect for `Exasol`_.
-
-.. _`Exasol`: https://www.exasol.com/
-
-.. _hive_dialect_ref:
-
-Hive
-----
-
-The dialect for `Hive`_.
-
-.. _`Hive`: https://hive.apache.org/
-
-.. _mysql_dialect_ref:
-
-MySQL
------
-
-The dialect for `MySQL`_.
-
-.. _`MySQL`: https://www.mysql.com/
-
-.. _oracle_dialect_ref:
-
-Oracle
-------
-
-The dialect for `Oracle`_ SQL. Note: this does not include PL/SQL.
-
-.. _`Oracle`: https://www.oracle.com/database/technologies/appdev/sql.html
-
-.. _postgres_dialect_ref:
-
-PostgreSQL
-----------
-
-This is based around the `PostgreSQL spec`_. Many other SQL instances are often
-based on PostreSQL syntax. If you're running an unsupported dialect, then
-this is often the dialect to use (until someone makes a specific dialect).
-
-.. _`PostgreSQL spec`: https://www.postgresql.org/docs/9.6/reference.html
-
-.. _redshift_dialect_ref:
-
-Redshift
-----------
-
-
-The dialect for `Amazon Redshift`_.
-
-.. _`Amazon Redshift`: https://aws.amazon.com/redshift/
-
-.. _snowflake_dialect_ref:
-
-Snowflake
----------
-
-The dialect for `Snowflake`_, which has much of its syntax
-inherited from :ref:`postgres_dialect_ref`.
-
-.. _`Snowflake`: https://docs.snowflake.com/en/sql-reference.html
-
-.. _spark3_dialect_ref:
-
-Spark3
-------
-
-The dialect for `Spark3`_. It inherits from :ref:`ansi_dialect_ref` and
-includes relevant syntax from :ref:`hive_dialect_ref` for commands that
-permit Hive Format.
-
-This implementation focuses on the `ANSI Compliant dialect`_ introduced in
-Spark3, instead of being Hive Compliant. The introduction of ANSI Compliance
-provides better data quality and easier migration from traditional DBMS.
-
-Versions of Spark prior to 3.x will only support the Hive dialect.
-
-.. _`Spark3`: https://spark.apache.org/docs/latest/sql-ref.html
-.. _`Ansi Compliant dialect`: https://spark.apache.org/docs/latest/sql-ref-ansi-compliance.html
-
-.. _sqlite_dialect_ref:
-
-SQLite
-------
-
-The dialect for `SQLite`_.
-
-.. _`SQLite`: https://www.sqlite.org/
-
-.. _tsql_dialect_ref:
-
-T-SQL
------
-
-The dialect for `T-SQL`_ (aka Transact-SQL).
-
-.. _`T-SQL`: https://docs.microsoft.com/en-us/sql/t-sql/language-reference
-
-.. _teradata_dialect_ref:
-
-Teradata
---------
-
-The dialect for `Teradata`_.
-
-.. _`Teradata`: https://www.teradata.co.uk/

docs/source/gettingstarted.rst

@@ -43,7 +43,9 @@ If however, you do have python installed but not :code:`pip`, then
 the best instructions for what to do next are `on the python website`_.
 
 .. _`instructions for all platforms here`: https://wiki.python.org/moin/BeginnersGuide/Download
-.. _`on the python website`: https://pip.pypa.io/en/stable/installing/
+.. _`on the python website`: https://pip.pypa.io/en/stable/installation/
+
+.. _installingsqlfluff:
 
 Installing SQLFluff
 -------------------
@@ -61,7 +63,7 @@ version number.
 .. code-block:: text
 
     $ sqlfluff version
-    0.3.1
+    3.2.5
 
 Basic Usage
 -----------
@@ -76,29 +78,40 @@ same folder that you're currently in with the following content:
     SELECT a+b  AS foo,
     c AS bar from my_table
 
-You can then run :code:`sqlfluff lint test.sql` to lint this file.
+You can then run :code:`sqlfluff lint test.sql --dialect ansi` to lint this
+file.
 
 .. code-block:: text
 
-    $ sqlfluff lint test.sql
+    $ sqlfluff lint test.sql --dialect ansi
     == [test.sql] FAIL
-    L:   1 | P:   1 | L036 | Select targets should be on a new line unless there is
+    L:   1 | P:   1 | LT09 | Select targets should be on a new line unless there is
                            | only one select target.
-    L:   1 | P:   8 | L034 | Use wildcards then simple select targets before
-                           | calculations and aggregates.
-    L:   1 | P:   9 | L006 | Missing whitespace before +
-    L:   1 | P:   9 | L006 | Missing whitespace after +
-    L:   1 | P:  11 | L039 | Unnecessary whitespace found.
-    L:   2 | P:   1 | L003 | Indent expected and not found compared to line #1
-    L:   2 | P:  10 | L010 | Keywords must be consistently upper case.
-    L:   2 | P:  15 | L009 | Files must end with a single trailing newline.
+                           | [layout.select_targets]
+    L:   1 | P:   1 | ST06 | Select wildcards then simple targets before calculations
+                           | and aggregates. [structure.column_order]
+    L:   1 | P:   7 | LT02 | Expected line break and indent of 4 spaces before 'a'.
+                           | [layout.indent]
+    L:   1 | P:   9 | LT01 | Expected single whitespace between naked identifier and
+                           | binary operator '+'. [layout.spacing]
+    L:   1 | P:  10 | LT01 | Expected single whitespace between binary operator '+'
+                           | and naked identifier. [layout.spacing]
+    L:   1 | P:  11 | LT01 | Expected only single space before 'AS' keyword. Found '
+                           | '. [layout.spacing]
+    L:   2 | P:   1 | LT02 | Expected indent of 4 spaces.
+                           | [layout.indent]
+    L:   2 | P:   9 | LT02 | Expected line break and no indent before 'from'.
+                           | [layout.indent]
+    L:   2 | P:  10 | CP01 | Keywords must be consistently upper case.
+                           | [capitalisation.keywords]
+    All Finished 📜 🎉!
 
 You'll see that *SQLFluff* has failed the linting check for this file.
 On each of the following lines you can see each of the problems it has
 found, with some information about the location and what kind of
 problem there is. One of the errors has been found on *line 1*, *position *
 (as shown by :code:`L:   1 | P:   9`) and it's a problem with rule
-*L006* (for a full list of rules, see :ref:`ruleref`). From this
+*LT01* (for a full list of rules, see :ref:`ruleref`). From this
 (and the following error) we can see that the problem is that there
 is no space either side of the :code:`+` symbol in :code:`a+b`.
 Head into the file, and correct this issue so that the file now
@@ -110,20 +123,27 @@ looks like this:
     c AS bar from my_table
 
 Rerun the same command as before, and you'll see that the original
-error (violation of *L006*) no longer shows up.
+error (violation of *LT01*) no longer shows up.
 
 .. code-block:: text
 
-    $ sqlfluff lint test.sql
+    $ sqlfluff lint test.sql --dialect ansi
     == [test.sql] FAIL
-    L:   1 | P:   1 | L036 | Select targets should be on a new line unless there is
+    L:   1 | P:   1 | LT09 | Select targets should be on a new line unless there is
                            | only one select target.
-    L:   1 | P:   8 | L034 | Use wildcards then simple select targets before
-                           | calculations and aggregates.
-    L:   1 | P:  13 | L039 | Unnecessary whitespace found.
-    L:   2 | P:   1 | L003 | Indent expected and not found compared to line #1
-    L:   2 | P:  10 | L010 | Keywords must be consistently upper case.
-    L:   2 | P:  15 | L009 | Files must end with a single trailing newline.
+                           | [layout.select_targets]
+    L:   1 | P:   1 | ST06 | Select wildcards then simple targets before calculations
+                           | and aggregates. [structure.column_order]
+    L:   1 | P:   7 | LT02 | Expected line break and indent of 4 spaces before 'a'.
+                           | [layout.indent]
+    L:   1 | P:  13 | LT01 | Expected only single space before 'AS' keyword. Found '
+                           | '. [layout.spacing]
+    L:   2 | P:   1 | LT02 | Expected indent of 4 spaces.
+                           | [layout.indent]
+    L:   2 | P:   9 | LT02 | Expected line break and no indent before 'from'.
+                           | [layout.indent]
+    L:   2 | P:  10 | CP01 | Keywords must be consistently upper case.
+                           | [capitalisation.keywords]
 
 To fix the remaining issues, we're going to use one of the more
 advanced features of *SQLFluff*, which is the *fix* command. This
@@ -133,18 +153,23 @@ and there may be some situations where a fix may not be able to be
 applied because of the context of the query, but in many simple cases
 it's a good place to start.
 
-For now, we only want to fix the following rules: *L003*, *L009*, *L010*
+For now, we only want to fix the following rules: *LT02*, *LT12*, *CP01*
 
 .. code-block:: text
 
-    $ sqlfluff fix test.sql --rules L003,L009,L010
+    $ sqlfluff fix test.sql --rules LT02,LT12,CP01 --dialect ansi
     ==== finding violations ====
     == [test.sql] FAIL
-    L:   2 | P:   1 | L003 | Indent expected and not found compared to line #1
-    L:   2 | P:  10 | L010 | Keywords must be consistently upper case.
-    L:   2 | P:  15 | L009 | Files must end with a single trailing newline.
+    L:   1 | P:   7 | LT02 | Expected line break and indent of 4 spaces before 'a'.
+                           | [layout.indent]
+    L:   2 | P:   1 | LT02 | Expected indent of 4 spaces.
+                           | [layout.indent]
+    L:   2 | P:   9 | LT02 | Expected line break and no indent before 'FROM'.
+                           | [layout.indent]
+    L:   2 | P:  10 | CP01 | Keywords must be consistently upper case.
+                           | [capitalisation.keywords]
     ==== fixing violations ====
-    3 fixable linting violations found
+    4 fixable linting violations found
     Are you sure you wish to attempt to fix these? [Y/n]
 
 ...at this point you'll have to confirm that you want to make the
@@ -163,33 +188,32 @@ now different.
 
 .. code-block:: sql
 
-    SELECT a + b  AS foo,
-        c AS bar FROM my_table
+    SELECT
+        a + b  AS foo,
+        c AS bar
+    FROM my_table
 
 In particular:
 
-* The second line has been indented to reflect being inside the
+* The two columns have been indented to reflect being inside the
   :code:`SELECT` statement.
 * The :code:`FROM` keyword has been capitalised to match the
   other keywords.
-* A final newline character has been added at the end of the
-  file (which may not be obvious in the snippet above).
 
 We could also fix *all* of the fixable errors by not
 specifying :code:`--rules`.
 
 .. code-block:: text
 
-    $ sqlfluff fix test.sql
+    $ sqlfluff fix test.sql --dialect ansi
     ==== finding violations ====
     == [test.sql] FAIL
-    L:   1 | P:   1 | L036 | Select targets should be on a new line unless there is
-                           | only one select target.
-    L:   1 | P:   8 | L034 | Use wildcards then simple select targets before
-                           | calculations and aggregates.
-    L:   1 | P:  13 | L039 | Unnecessary whitespace found.
+    L:   1 | P:   1 | ST06 | Select wildcards then simple targets before calculations
+                           | and aggregates. [structure.column_order]
+    L:   2 | P:  10 | LT01 | Expected only single space before 'AS' keyword. Found '
+                           | '. [layout.spacing]
     ==== fixing violations ====
-    3 fixable linting violations found
+    2 fixable linting violations found
     Are you sure you wish to attempt to fix these? [Y/n] ...
     Attempting fixes...
     Persisting Changes...
@@ -203,7 +227,8 @@ been updated again.
 
     SELECT
         c AS bar,
-        a + b AS foo FROM my_table
+        a + b AS foo
+    FROM my_table
 
 The SQL statement is now well formatted according to all the
 rules defined in SQLFluff.
@@ -229,17 +254,20 @@ put the following content:
 
 .. code-block:: cfg
 
-    [sqlfluff:rules]
+    [sqlfluff]
+    dialect = ansi
+
+    [sqlfluff:indentation]
     tab_space_size = 2
 
-    [sqlfluff:rules:L010]
+    [sqlfluff:rules:capitalisation.keywords]
     capitalisation_policy = lower
 
 Then rerun the same command as before.
 
 .. code-block:: text
 
-    $ sqlfluff fix test.sql --rules L003,L009,L010,L034,L036,L039
+    $ sqlfluff fix test.sql --rules LT02,LT12,CP01,ST06,LT09,LT01
 
 Then examine the file again, and you'll notice that the
 file has been fixed accordingly.
@@ -247,10 +275,13 @@ file has been fixed accordingly.
 .. code-block:: sql
 
     select
-        c as bar,
-        a + b as foo from my_table
+      c as bar,
+      a + b as foo
+    from my_table
 
 For a full list of configuration options check out :ref:`defaultconfig`.
+Note that in our example here we've only set a few configuration values
+and any other configuration settings remain as per the default config.
 To see how these options apply to specific rules check out the
 "Configuration" section within each rule's documentation in :ref:`ruleref`.
 
@@ -270,6 +301,8 @@ From here, there are several more things to explore.
 * To find out more about which rules are available, see :ref:`ruleref`.
 * To find out more about configuring *SQLFluff* and what other options
   are available, see :ref:`config`.
+* Once you're ready to start using *SQLFluff* on a project or with the
+  rest of your team, check out :ref:`production-use`.
 
 One last thing to note is that *SQLFluff* is a relatively new project
 and you may find bugs or strange things while using it. If you do find

docs/source/guides/contributing/architecture.rst

@@ -0,0 +1,131 @@
+.. _architecture:
+
+Architecture
+------------
+
+At a high level, the behaviour of SQLFluff is divided into a few key stages.
+Whether calling `sqlfluff lint`, `sqlfluff fix` or `sqlfluff parse`, the
+internal flow is largely the same.
+
+.. _templater:
+
+Stage 1, the templater
+^^^^^^^^^^^^^^^^^^^^^^
+
+This stage only applies to templated SQL. Vanilla SQL is sent straight to
+stage 2, the lexer.
+
+In order to lint templated SQL, SQLFluff must first convert the 'raw' or
+pre-templated code into valid SQL, which can then be parsed. The templater
+returns both the raw and post-templated SQL so that any rule violations which
+occur in templated sections can be ignored and the rest mapped to their
+original line location for user feedback.
+
+.. _Jinja: https://jinja.palletsprojects.com/
+.. _dbt: https://docs.getdbt.com/
+.. _`Python format strings`: https://docs.python.org/3/library/string.html#format-string-syntax
+
+*SQLFluff* supports multiple templating engines:
+
+   * Jinja_
+   * SQL placeholders (e.g. SQLAlchemy parameters)
+   * `Python format strings`_
+   * dbt_ (via plugin)
+
+Under the hood dbt also uses Jinja, but in *SQLFluff* uses a separate
+mechanism which interfaces directly with the dbt python package.
+
+For more details on how to configure the templater see :ref:`templateconfig`.
+
+
+Stage 2, the lexer
+^^^^^^^^^^^^^^^^^^
+
+The lexer takes SQL and separates it into segments of whitespace and
+code. Where we can impart some high level meaning to segments, we
+do, but the result of this operation is still a flat sequence of
+typed segments (all subclasses of :code:`RawSegment`).
+
+
+Stage 3, the parser
+^^^^^^^^^^^^^^^^^^^
+
+The parser is arguably the most complicated element of SQLFluff, and is
+relied on by all the other elements of the tool to do most of the heavy
+lifting.
+
+#. The lexed segments are parsed using the specified dialect's grammars. In
+   SQLFluff, grammars describe the shape of SQL statements (or their
+   components). The parser attempts to apply each potential grammar to the
+   lexed segments until all the segments have been matched.
+
+#. In SQLFluff, segments form a tree-like structure. The top-level segment is
+   a :code:`FileSegment`, which contains zero or more
+   :code:`StatementSegment`\ s, and so on. Before the segments have been parsed
+   and named according to their type, they are 'raw', meaning they have no
+   classification other than their literal value.
+
+#. A segment's :code:`.match()` method uses the :code:`match_grammar`, on
+   which :code:`.match()` is called. SQLFluff parses in a single pass through
+   the file, so segments will recursively match the file based on their
+   respective grammars. In the example of a :code:`FileSegment`, it
+   first divides up the query into statements, and then the :code:`.match()`
+   method of those segments works out the structure within them.
+
+   * *Segments* must implement a :code:`match_grammar`. When :code:`.match()`
+      is called on a segment, this is the grammar which is used to decide
+      whether there is a match.
+
+   * *Grammars* combine *segments* or other *grammars* together in a
+      pre-defined way. For example the :code:`OneOf` grammar will match if any
+      one of its child elements match.
+
+   #. During the recursion, the parser eventually reaches segments which have
+      no children (raw segments containing a single token), and so the
+      recursion naturally finishes.
+
+#. If no match is found for a segment, the contents will be wrapped in an
+   :code:`UnparsableSegment` which is picked up as a *parsing* error later.
+   This is usually facilitated by the :code:`ParseMode` on some grammars
+   which can be set to :code:`GREEDY`, allowing the grammar to capture
+   additional segments as unparsable. As an example, bracketed sections
+   are often configured to capture anything unexpected as unparsable rather
+   than simply failing to match if there is more than expected (which would
+   be the default, :code:`STRICT`, behaviour).
+
+#. The result of the :code:`.match()` method is a :code:`MatchResult` which
+   contains the instructions on how to turn the flat sequence of raw segments
+   into a nested tree of segments. Calling :code:`.apply()` on this result
+   at the end of the matching process is what finally creates the nested
+   structure.
+
+When working on the parser there are a couple of design principles
+to keep in mind.
+
+- Grammars are contained in *dialects*, the root dialect being
+  the *ansi* dialect. The ansi dialect is used to host logic common
+  to all dialects, and so does not necessarily adhere to the formal
+  ansi specification. Other SQL dialects inherit from the ansi dialect,
+  replacing or patching any segments they need to. One reason for the *Ref*
+  grammar is that it allows name resolution of grammar elements at runtime
+  and so a *patched* grammar with some elements overridden can still rely on
+  lower-level elements which haven't been redeclared within the dialect
+- All grammars and segments attempt to match as much as they can and will
+  return partial matches where possible. It is up to the calling grammar
+  or segment to decide whether a partial or complete match is required
+  based on the context it is matching in.
+
+
+Stage 4, the linter
+^^^^^^^^^^^^^^^^^^^
+
+Given the complete parse tree, rule classes check for linting errors by
+traversing the tree, looking for segments and patterns of concern. If
+the rule discovers a violation, it returns a
+:py:class:`~sqlfluff.core.rules.base.LintResult` pointing to the segment
+which caused the violation.
+
+Some rules are able to *fix* the problems they find. If this is the case,
+the rule will return a list of fixes, which describe changes to be made to
+the tree. This can include edits, inserts, or deletions. Once the fixes
+have been applied, the updated tree is written to the original file.

docs/source/guides/contributing/docs.rst

@@ -0,0 +1,121 @@
+Documentation Contributions
+===========================
+
+Contributing to the docs is one of the easiest and most helpful ways
+to help the project. Documentation changes require relatively little
+specialist knowledge apart from being familiar with how to use SQLFluff
+and the docs are read by a very wide range of people.
+
+Documentation takes two forms:
+
+1. Embedded documentation found in function and module `docstrings`_.
+
+2. The free-standing documentation which you're reading now, and hosted
+   at `docs.sqlfluff.com`_ (built using `sphinx`_ and `ReadtheDocs`_).
+
+The two are somewhat blurred by the use of `autodoc`_ (and some other custom
+integrations), where documentation is generated directly off `docstrings`_
+within the codebase, for example the :ref:`ruleref`, :ref:`cliref` and
+:ref:`dialectref`. To understand more about how the custom integrations
+we use to generate these docs, see the `generate-auto-docs.py`_ file.
+
+.. _`docstrings`:  https://en.wikipedia.org/wiki/Docstring
+.. _`docs.sqlfluff.com`:  https://docs.sqlfluff.com
+.. _`autodoc`: https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html
+.. _`generate-auto-docs.py`: https://github.com/sqlfluff/sqlfluff/blob/main/docs/generate-auto-docs.py
+
+.. _`ReadtheDocs`: https://about.readthedocs.com/
+
+Docstrings
+----------
+
+Embedded documentation of functions, classes and modules is most useful
+for *developer-focussed* documentation as it's most accessible in the places
+which those developers are working: *directly in the codebase*. We enforce
+that docstrings are present and correctly formatted using the
+`pydocstyle rules for ruff`_, which we have configured to enforce the
+`google style of docstrings`_.
+
+.. _`pydocstyle rules for ruff`:  https://docs.astral.sh/ruff/rules/#pydocstyle-d
+.. _`google style of docstrings`:  https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html
+
+Sphinx Docs
+-----------
+
+The main documentation (which you're reading now), is build using `sphinx`_,
+and written using `reStructuredText`_ (files ending with :code:`.rst`). The
+`sphinx`_ project offers a `reStructuredText primer`_ for people who are new
+to the syntax (and the SQLFluff project uses `doc8`_ in the CI process to try
+and catch any issues early).
+
+On top of those docs, there are a few areas worth highlighting for new (or
+returning) users, which are either specific to the SQLFluff project, or not
+particularly clear in the sphinx docs:
+
+* `reStructuredText`_ is very similar to, but differs from (the somewhat more
+  well known) `Markdown`_ syntax. Importantly:
+
+  * :code:`*text with single asterisks*` renders as *italics*. Use
+    :code:`**double asterisks**` for **bold text**.
+
+  * :code:`code snippets` are created using the |codesnippet|
+    directive, rather than just lone backticks (|backquotes|) as found in
+    most `Markdown`_.
+
+* To create links to other parts of the documentation (i.e.
+  `Cross-referencing`_), use either the :code:`:ref:` syntax.
+
+  * Docs for all the SQL dialects are auto generated with associated anchors
+    to use for referencing. For example to link to the
+    :ref:`postgres_dialect_ref` dialect docs, you can use the |postgresref|.
+    Replace the :code:`postgres` portion with the :code:`name` of the
+    dialect you want to link to.
+
+  * Docs for all the bundled rules and handled using a customer `sphinx`_
+    plugin, which means you can refer to them using their name or code:
+    |LT01ref| resolves to :sqlfluff:ref:`LT01` and |layoutspacingref|
+    resolves to :sqlfluff:ref:`layout.spacing`.
+
+  * Docs for any of the python classes and modules handled using `autodoc`_
+    can be referenced as per their docs, so the
+    :py:class:`sqlfluff.core.rules.base.BaseRule` class can be referenced
+    with |baseruleref|. You can also use the :code:`~` prefix (i.e.
+    |shortbaseruleref|) so that it just renders as
+    :py:class:`~sqlfluff.core.rules.base.BaseRule`. See the docs for
+    `Cross-referencing`_ for more details.
+
+.. _`sphinx`: https://www.sphinx-doc.org/en/master/
+.. _`reStructuredText`: https://www.sphinx-doc.org/en/master/usage/restructuredtext/index.html
+.. _`reStructuredText primer`: https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html
+.. _`doc8`: https://github.com/PyCQA/doc8
+.. _`Markdown`: https://www.markdownguide.org/
+.. _`Cross-referencing`: https://www.sphinx-doc.org/en/master/usage/referencing.html
+.. _`autodoc`: https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html
+
+.. |codesnippet| raw:: html
+
+    <code class="code docutils literal notranslate">:code:`...`</code>
+
+.. |backquotes| raw:: html
+
+    <code class="code docutils literal notranslate">`...`</code>
+
+.. |postgresref| raw:: html
+
+    <code class="code docutils literal notranslate">:ref:`postgres_dialect_ref`</code>
+
+.. |LT01ref| raw:: html
+
+    <code class="code docutils literal notranslate">:sqlfluff:ref:`LT01`</code>
+
+.. |layoutspacingref| raw:: html
+
+    <code class="code docutils literal notranslate">:sqlfluff:ref:`layout.spacing`</code>
+
+.. |baseruleref| raw:: html
+
+    <code class="code docutils literal notranslate">:py:class:`sqlfluff.core.rules.base.BaseRule`</code>
+
+.. |shortbaseruleref| raw:: html
+
+    <code class="code docutils literal notranslate">:py:class:`~sqlfluff.core.rules.base.BaseRule`</code>

docs/source/guides/contributing/plugins.rst

@@ -0,0 +1,96 @@
+.. _developingpluginsref:
+
+Developing Plugins
+==================
+
+*SQLFluff* is extensible through "plugins". We use the `pluggy library`_
+to make linting Rules pluggable, which enable users to implement rules that
+are just too "organization specific" to be shared, or too platform specific
+to be included in the core library.
+
+.. note::
+
+    We recommend that the module in a plugin which defines all
+    of the hook implementations (anything using the ``@hookimpl`` decorator)
+    must be able to fully import before any rule implementations are imported.
+    More specifically, SQLFluff must be able to both *import* **and**
+    *run* any implementations of ``get_configs_info()`` before any plugin
+    rules (i.e. any derivatives of :py:class:`~sqlfluff.core.rules.base.BaseRule`)
+    are *imported*. Because of this, we recommend that rules are defined in a
+    separate module to the root of the plugin and then only imported within
+    the ``get_rules()`` method.
+
+    Importing in the main body of the module was previously our recommendation
+    and so may be the case for versions of some plugins. If one of your plugins
+    does use imports in this way, a warning will be presented, recommending that
+    you update your plugin.
+
+    .. code-block:: python
+       :emphasize-lines: 7,8
+
+        # The root module will need to import `hookimpl`, but
+        # should not yet import the rule definitions for the plugin.
+        from sqlfluff.core.plugin import hookimpl
+
+        @hookimpl
+        def get_rules():
+            # Rules should be imported within the `get_rules` method instead
+            from my_plugin.rules import MyRule
+            return [MyRule]
+
+
+.. _`pluggy library`: https://pluggy.readthedocs.io/en/latest/
+
+Creating a plugin
+-----------------
+
+We have an example plugin in
+`sqlfluff/plugins/sqlfluff-plugin-example`_ which you can use as a template
+for rules, or the `sqlfluff/plugins/sqlfluff-templater-dbt`_ which you can
+use as a template for templater plugins.
+
+Few things to note about plugins:
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Currently, only Rules and Templaters can be added through plugins. Over time
+we expect more elements of SQLFluff will be extensible with plugins. Each
+plugin can implement multiple Rules or Templaters.
+
+We recommend that the name of a plugin should start with *"sqlfluff-"* to be
+clear on the purpose of your plugin.
+
+A plugin may need to include a default configuration if its rules
+are configurable: use plugin default configurations **only for that reason**!
+We advise against overwriting core configurations by using a default
+plugin configuration, as there is no mechanism in place to enforce precedence
+between the core library configs and plugin configs,
+and multiple plugins could clash.
+
+A plugin Rule class name should have the structure:
+"Rule_PluginName_L000". The 'L' can be any letter
+and is meant to categorize rules; you could use the
+letter 'S' to denote rules that enforce security checks
+for example.
+
+An important thing to note when running custom implemented rules:
+Run ``pip install -e .``, inside the plugin folder so custom rules in linting
+are included.
+
+A plugin Rule code includes the PluginName,
+so a rule "Rule_L000" in core will have code "L000",
+while "Rule_PluginName_L000" will have code "PluginName_L000".
+Codes are used to display errors, they are also used as configuration keys.
+
+We make it easy for plugin developers to test their rules by
+exposing a testing library in *sqlfluff.utils.testing*.
+
+.. _`sqlfluff/plugins/sqlfluff-plugin-example`: https://github.com/sqlfluff/sqlfluff/tree/main/plugins/sqlfluff-plugin-example
+.. _`sqlfluff/plugins/sqlfluff-templater-dbt`: https://github.com/sqlfluff/sqlfluff/tree/main/plugins/sqlfluff-templater-dbt
+
+Giving feedback
+---------------
+
+Would you like to have other parts of *SQLFluff* be "pluggable"?
+Tell us about it in a `GitHub issue`_ 😄.
+
+.. _`GitHub issue`: https://github.com/sqlfluff/sqlfluff/issues/new?assignees=&labels=enhancement&template=enhancement.md

docs/source/guides/contributing/rules.rst

@@ -0,0 +1,99 @@
+.. _developingrulesref:
+
+Developing Rules
+================
+
+`Rules` in `SQLFluff` are implemented as classes inheriting from ``BaseRule``.
+SQLFluff crawls through the parse tree of a SQL file, calling the rule's
+``_eval()`` function for each segment in the tree. For many rules, this allows
+the rule code to be really streamlined and only contain the logic for the rule
+itself, with all the other mechanics abstracted away.
+
+Running Tests
+-------------
+
+The majority of the test cases for most bundled rules are *"yaml test cases"*,
+i.e. test cases defined in `yaml`_ files. You can find those `yaml fixtures on github`_.
+While this provides a very simple way to *write* tests, it can be occasionally tedious
+to *run* specific tests.
+
+Within either a `tox` environment or `virtualenv` (as described in the `contributing.md`_
+file), you can either run all of the rule yaml tests with:
+
+.. code-block:: sh
+
+   pytest test/rules/yaml_test_cases_test.py -vv
+
+...or to just run tests for a specific rule, there are two options for a syntax to select
+only those tests:
+
+.. code-block:: sh
+
+   pytest -vv test/rules/ -k RF01
+
+The :code:`-k` option simply searches for the content of the argument being in the name
+of the test, which will match any single or combo tests for that rule. By convention,
+any test cases for a rule should include the code for that rule.
+
+.. _`yaml`: https://yaml.org/
+.. _`yaml fixtures on github`: https://github.com/sqlfluff/sqlfluff/tree/main/test/fixtures/rules/std_rule_cases
+.. _`contributing.md`: https://github.com/sqlfluff/sqlfluff/blob/main/CONTRIBUTING.md
+
+Traversal Options
+-----------------
+
+``recurse_into``
+^^^^^^^^^^^^^^^^
+Some rules are a poor fit for the simple traversal pattern described above.
+Typical reasons include:
+
+* The rule only looks at a small portion of the file (e.g. the beginning or
+  end).
+* The rule needs to traverse the parse tree in a non-standard way.
+
+These rules can override ``BaseRule``'s ``recurse_into`` field, setting it to
+``False``. For these rules ``False``, ``_eval()`` is only called *once*, with
+the root segment of the tree. This can be much more efficient, especially on
+large files. For example, see rules ``LT13`` and ``LT12`` , which only look at
+the beginning or end of the file, respectively.
+
+``_works_on_unparsable``
+^^^^^^^^^^^^^^^^^^^^^^^^
+By default, `SQLFluff` calls ``_eval()`` for all segments, even "unparsable"
+segments, i.e. segments that didn't match the parsing rules in the dialect.
+This causes issues for some rules. If so, setting ``_works_on_unparsable``
+to ``False`` tells SQLFluff not to call ``_eval()`` for unparsable segments and
+their descendants.
+
+Performance-related Options
+---------------------------
+These are other fields on ``BaseRule``. Rules can override them.
+
+``needs_raw_stack``
+^^^^^^^^^^^^^^^^^^^
+``needs_raw_stack`` defaults to ``False``. Some rules use
+``RuleContext.raw_stack`` property to access earlier segments in the traversal.
+This can be useful, but it adds significant overhead to the linting process.
+For this reason, it is disabled by default.
+
+``lint_phase``
+^^^^^^^^^^^^^^
+There are two phases of rule running.
+
+1. The ``main`` phase is appropriate for most rules. These rules are assumed to
+interact and potentially cause a cascade of fixes requiring multiple passes.
+These rules run the `runaway_limit` number of times (default 10).
+
+2. The ``post`` phase is for post-processing rules, not expected to trigger
+any downstream rules, e.g. capitalization fixes. They are run in a
+post-processing loop at the end. This loop is identical to the ``main`` loop,
+but is only run 2 times at the end (once to fix, and once again to confirm no
+remaining issues).
+
+The two phases add complexity, but they also improve performance by allowing
+SQLFluff to run fewer rules during the ``main`` phase, which often runs several
+times.
+
+NOTE: ``post`` rules also run on the *first* pass of the ``main`` phase so that
+any issues they find will be presented in the list of issues output by
+``sqlfluff fix`` and ``sqlfluff lint``.

docs/source/guides/index.rst

@@ -0,0 +1,47 @@
+.. _guides:
+
+Guides & How-tos
+================
+
+This section is of short guides and articles is meant to be read alongside
+the rest of the documentation which is more reference-oriented.
+
+Setting up SQLFluff
+-------------------
+
+.. toctree::
+   :maxdepth: 1
+
+   setup/teamrollout
+   setup/developing_custom_rules
+
+Troubleshooting SQLFluff
+------------------------
+
+.. toctree::
+   :maxdepth: 1
+
+   troubleshooting/how_to
+
+.. _development:
+
+Contributing to SQLFluff
+------------------------
+
+It is recommended that the following is read in conjunction with exploring
+the codebase. `dialect_ansi.py`_ in particular is helpful to understand the
+recursive structure of segments and grammars.
+
+You may also need to reference the :ref:`internal_api_docs`.
+
+.. _`dialect_ansi.py`: https://github.com/sqlfluff/sqlfluff/blob/main/src/sqlfluff/dialects/dialect_ansi.py
+
+.. toctree::
+   :maxdepth: 1
+
+   contributing/architecture
+   contributing/git
+   contributing/dialect
+   contributing/rules
+   contributing/plugins
+   contributing/docs

docs/source/guides/setup/developing_custom_rules.rst

@@ -0,0 +1,134 @@
+.. _developing_custom_rules:
+
+Developing Custom Rules
+=======================
+
+It's quite common to have organisation-, or project-specific norms and
+conventions you might want to enforce using SQLFluff. With a little bit
+of python knowledge this is very achievable with SQLFluff, and there's
+a plugin architecture to support that.
+
+This guide should be read alongside the code for the
+`SQLFluff example plugin`_ and the more technical documentation for
+:ref:`developingpluginsref`.
+
+What Plugin do I need?
+----------------------
+
+When thinking about developing a rule, the following thought process will
+help you decide what to develop:
+
+1. When do I want this rule to show a warning, when should it definitely
+   **not** show one? What information do I need when evaluating whether
+   a the rule has been followed or not? This information will tell you
+   about the two important *locations* in the parse tree which will become
+   important.
+
+   * The *trigger* location: i.e. when should the rule be *called* for
+     evaluation. e.g. :sqlfluff:ref:`CP01` triggers on keywords, because
+     it only needs the information about that keyword to run, but
+     :sqlfluff:ref:`LT08` triggers on ``WITH`` statements even though it's
+     only interested in specific pieces of whitespace, because it needs the
+     full context of the statement to evaluate. You may with to examine the
+     parse structure of some example queries you'd want to handle by using
+     ``sqlfluff parse my_file.sql`` to identify the right segment. This is
+     then specified using the ``crawl_behaviour`` attribute on the rule.
+
+   * The *anchor* location: i.e. which position will show up in the CLI
+     readout back to the user. To continue the example of above, while
+     :sqlfluff:ref:`LT08` *triggers* on a ``WITH`` statement, it *anchors*
+     on a more specific segment just after where it expected whitespace.
+     It specifies this using the ``anchor`` argument to the
+     :py:class:`~sqlfluff.core.rules.base.LintResult` object.
+
+2. How should the rule evaluate and should I implement an auto-fix? For
+   the simplest rules, it the logic to evaluate whether there's an issue
+   can be *very simple*. For example in the `SQLFluff example plugin`_,
+   we are just checking the name of an element isn't in a configured list.
+   Typically we recommend that for organisation-specific rules, **KEEP IT**
+   **SIMPLE**. Some of the rules bundled with SQLFluff contain a lot of
+   complexity for handling how to automatically fix lots of edge cases,
+   but for your organisation it's probably not worth the overhead unless
+   you're a **very big team** or **come across a huge amount of poorly**
+   **formatted SQL**.
+
+   * Consider the information not just to *trigger*, but also whether a
+     custom error message would be appropriate and how to get the information
+     to construct that too. The default error message will be the first
+     line of the rule docstring_. Custom messages can be configured by
+     setting the ``description`` argument of the
+     :py:class:`~sqlfluff.core.rules.base.LintResult` object.
+
+   * Do use the existing SQLFluff core rules as examples of what is possible
+     and how to achieve various things - but remember that many of them
+     implement a level of complexity and edge case handling which may not
+     be necessary for your organisation.
+
+3. How am I going to roll out my rule to the team? Thinking through this
+   aspect of rule development is just as important as the technical aspect.
+   Spending a lot of time on rule development for it to be rejected by
+   the end users of it is both a waste of time and also counterproductive.
+
+   * Consider manually fixing any pre-existing issues in your project which
+     would trigger the rule before rollout.
+
+   * Seek consensus on how strictly the rule will be enforced and what the
+     step by step pathway is to strict enforcement.
+
+   * Consider *beta-testing* your new rule with a smaller group of users
+     who are more engaged with SQLFluff or code quality in general.
+
+.. _docstring:  https://en.wikipedia.org/wiki/Docstring
+
+Plugin Discovery
+----------------
+
+One of most common questions asked with respect to custom plugins is
+*discovery*, or *"how do I tell SQLFluff where my plugin is"*. SQLFluff
+uses pluggy_ as it's plugin architecture (developed by the folks at pytest_).
+Pluggy uses the python packaging metadata for plugin discovery. This means
+that **your plugin must be installed as a python package for discovery**.
+Specifically, it must define an `entry point`_ for SQLFluff.
+When SQLFluff runs, it inspects installed python packages for this entry
+point and then can run any which define one. For example you'll see in the
+`SQLFluff example plugin`_ that the ``pyproject.toml`` file has the
+following section:
+
+.. code-block:: toml
+
+   [project.entry-points.sqlfluff]
+   # Change this name in your plugin, e.g. company name or plugin purpose.
+   sqlfluff_example = "sqlfluff_plugin_example"
+
+You can find equivalent examples for ``setup.cfg`` and ``setup.py`` in the
+python docs for `entry point`_. This information is registered
+*on install* of your plugin, (i.e. when running `pip install`, or equivalent
+if you're using a different package manager) so if you change it later, you
+may need to re-install your plugin.
+
+You can test whether your rule has been successfully discovered by running
+``sqlfluff rules`` and reviewing whether your new rule has been included in
+the readout.
+
+.. note::
+    If you're struggling with rule discovery, **use the example plugin**.
+    It can be much easier to take a known working example and then modify
+    from there:
+
+    1. Copy the code from the `SQLFluff example plugin`_ into a local
+       folder.
+
+    2. Run `pip install -e /path/to/where/you/put/it`.
+
+    3. Run `sqlfluff rules`, to confirm that the example plugin is present
+       to demonstrate to yourself that discovery is functional.
+
+    4. Then edit the example plugin to do what you want now that discovery
+       isn't an issue. You may have to re-run `pip install ...` if you
+       change anything in the rule metadata (like the entry point, filenames
+       or plugin location).
+
+.. _pluggy: https://pluggy.readthedocs.io/en/latest/
+.. _pytest: https://docs.pytest.org/en/stable/
+.. _`entry point`: https://setuptools.pypa.io/en/stable/userguide/entry_point.html
+.. _`SQLFluff example plugin`: https://github.com/sqlfluff/sqlfluff/tree/main/plugins/sqlfluff-plugin-example

docs/source/guides/setup/teamrollout.rst

@@ -3,12 +3,12 @@
 Rolling out SQLFluff with a new team
 ====================================
 
-Rolling out SQLFluff (like rolling out any other linter or style
-guide) is not just about just the *technical* rollout, but also
-how you introduce to the tool to the team and organisation around
+Rolling out SQLFluff, like rolling out any other linter or style
+guide, is not just about the *technical* rollout, but also
+how you introduce the tool to the team and organisation around
 you.
 
-   *The effect of SQLFluff should be to change your behaviours not*
+   *The effect of SQLFluff should be to change your behaviours, not*
    *just your SQL*.
 
 With that in mind, it's worth reminding ourselves what we're trying
@@ -25,7 +25,7 @@ might be:
    found in open source projects is easy to read and *looks familiar*
    then you're more likely to use it. This means more reusable code
    across the industry.
-#. **Productive discussions  around style**. By defining your
+#. **Productive discussions around style**. By defining your
    organisation's style guide in code, it means you can version
    control it, discuss changes and ultimately give a concrete output
    to discussions over style.
@@ -37,7 +37,7 @@ Consider which of these success measures is most important and most
 desirable for your team. *Write that down*.
 
 The following steps are a guide, which you should adapt to your
-organisation, and in particular it's level of data maturity.
+organisation, and in particular its level of data maturity.
 
 1. Assess the situation
 -----------------------
@@ -48,12 +48,12 @@ This step is done by you, or a small group of people who *already*
 * Run ``sqlfluff lint`` on your project with the stock configuration
   to find out how things work *out of the box*.
 * Set up your :ref:`config` so that things run and that you can get
-  a readout of of the errors which you would want the team to see and
+  a readout of the errors which you would want the team to see and
   *not the ones you don't*. Great tools for this are to use
   :ref:`sqlfluffignore`, ``--exclude-rules`` or ``--ignore`` in the
   CLI (see :ref:`cliref`).
-* Identify which areas of your project are the worst and which the
-  tidiest are. In particular, any areas which are particularly good
+* Identify which areas of your project are the worst and which are the
+  tidiest. In particular, any areas which are particularly tidy
   already will be particularly useful in the next phase.
 
 2. Make a plan
@@ -67,24 +67,24 @@ There are three sensible rollout phases:
 
 In each of these phases you have three levers to play with:
 
-#. Areas of the project.
+#. Areas of the project in which to apply rules.
 #. Depth of rules enforced (this might also include whether
-   to ignore parsing errors or not too).
+   to ignore parsing errors or not).
 #. Whether to just lint changes (:ref:`diff-quality`),
-   or whether to lint all the existing code as well.
+   or to lint all the existing code as well.
 
 Work out a sensible roadmap of how hard you want to go in
 each phase. Be clear who is responsible for changes at each
 phase. An example plan might look like this:
 
-#. **Pre CI/CD** we get the cleanest area of a project
-   to a stage that it fully passes the rules we eventually.
+#. **Pre CI/CD** we get the tidiest area of a project
+   to a stage that it fully passes the rules we eventually want to enforce.
    The core project team will do this. Liberal use of
    ``sqlfluff fix`` can be a lifesaver in this phase.
 #. **Soft CI/CD** is applied to the whole project, team
-   members are encouraged to write tidy SQL, but not *required*.
+   members are encouraged to write tidy SQL, but not *required* to.
 #. **Hard CI/CD** is applied to the tidy areas of the project
-   and to also to any changes to the whole project. Anyone
+   and also to any changes to the whole project. Anyone
    making changes is *required* to write SQL which passes check.
 #. **Hard CI/CD** is applied to the whole project on not just
    changes, with only a few particularly problematic files
@@ -103,10 +103,10 @@ this an empowering experience that everyone can get involved with
 rather than *another piece of admin they need to do*.
 
 At this stage, you might also want to consider other tools in the
-SQLFluff ecosystem such as the `SQLFluff pre-commit hook`_ and
-the `SQLFluff VSCode plugin`_ or `SQLFluff online formatter`_.
+SQLFluff ecosystem such as the :ref:`SQLFluff pre-commit hook
+<using-pre-commit>` and the `SQLFluff VSCode plugin`_ or `SQLFluff
+online formatter`_.
 
-.. _`SQLFluff pre-commit hook`: https://github.com/sqlfluff/sqlfluff-github-actions
 .. _`SQLFluff VSCode plugin`: https://github.com/sqlfluff/vscode-sqlfluff
 .. _`SQLFluff online formatter`: https://online.sqlfluff.com/
 
@@ -122,8 +122,8 @@ norms and rituals around how you change the rules. In particular:
   or whether some are creating unnecessary stress?
 
 It's normal for your usage of tools like SQLFluff to change and
-evolve over time, what's important is to know this in advance
-and welcome it when it happens, and make sure you're always driving
+evolve over time. It's important to expect this change in advance,
+and welcome it when it happens. Always make sure you're driving
 toward the success measures you decided up front, rather than
 just resisting the change.
 

docs/source/guides/troubleshooting/how_to.rst

@@ -0,0 +1,144 @@
+How to Troubleshoot SQLFluff
+============================
+
+It can at times be tricky to troubleshoot SQLFluff as it exists within
+an ecosystem of other tools, and can be deployed in wide range of ways.
+
+This step by step guide can help you narrow down what's likely going wrong
+and point you toward the swiftest resolution.
+
+1. Common Errors
+----------------
+
+There are a few error messages you may get which have relatively
+straightforward resolution paths.
+
+Parsing Errors
+^^^^^^^^^^^^^^
+
+SQLFluff needs to be able to parse your SQL to understand it's structure.
+That means if it fails to parse the SQL it will give you an error message.
+The intent is that if SQLFluff cannot parse the SQL, then it should mean
+the SQL is also invalid and help you understand where and why.
+
+For example, this is a simple query which is not valid SQL:
+
+.. code-block:: sql
+
+   select 1 2 3
+   from my_table
+
+When running ``sqlfluff lint`` or ``sqlfluff parse`` we get the following
+error message::
+
+    ==== parsing violations ====
+    L:   1 | P:  10 |  PRS | Line 1, Position 10: Found unparsable section: '2 3'
+
+Furthermore if we look at the full parsing output we can see an unparsable
+section in the parse tree:
+
+.. code-block::
+   :emphasize-lines: 12,13,14,15
+
+   [L:  1, P:  1]      |file:
+   [L:  1, P:  1]      |    statement:
+   [L:  1, P:  1]      |        select_statement:
+   [L:  1, P:  1]      |            select_clause:
+   [L:  1, P:  1]      |                keyword:                                      'select'
+   [L:  1, P:  7]      |                [META] indent:
+   [L:  1, P:  7]      |                whitespace:                                   ' '
+   [L:  1, P:  8]      |                select_clause_element:
+   [L:  1, P:  8]      |                    numeric_literal:                          '1'
+   [L:  1, P:  9]      |                [META] dedent:
+   [L:  1, P:  9]      |                whitespace:                                   ' '
+   [L:  1, P: 10]      |                unparsable:                                   !! Expected: 'Nothing here.'
+   [L:  1, P: 10]      |                    numeric_literal:                          '2'
+   [L:  1, P: 11]      |                    whitespace:                               ' '
+   [L:  1, P: 12]      |                    numeric_literal:                          '3'
+   [L:  1, P: 13]      |            newline:                                          '\n'
+   [L:  2, P:  1]      |            from_clause:
+   [L:  2, P:  1]      |                keyword:                                      'from'
+   [L:  2, P:  5]      |                whitespace:                                   ' '
+   [L:  2, P:  6]      |                from_expression:
+   [L:  2, P:  6]      |                    [META] indent:
+   [L:  2, P:  6]      |                    from_expression_element:
+   [L:  2, P:  6]      |                        table_expression:
+   [L:  2, P:  6]      |                            table_reference:
+   [L:  2, P:  6]      |                                naked_identifier:             'my_table'
+   [L:  2, P: 14]      |                    [META] dedent:
+   [L:  2, P: 14]      |    newline:                                                  '\n'
+   [L:  3, P:  1]      |    [META] end_of_file:
+
+SQLFluff maintains it's own version of each SQL dialect, and this may not be
+exhaustive for some of the dialects which are newer to SQLFluff or which are
+in very active development themselves. This means in some scenarios you may
+find a query which runs fine in your environment, but cannot be parsed by
+SQLFluff. This is not a *"bug"* per-se, but is an indicator of a gap in the
+SQLFluff dialect.
+
+Many of the issues raised on GitHub relate to parsing errors like this, but
+it's also a great way to support the project if you feel able to contribute
+a dialect improvement yourself. We have a short guide on
+:ref:`contributing_dialect_changes` to walk you through the process. In the
+short term you can also ignore specific files from your overall project so
+that this specific file doesn't become a blocker for the rest.
+See :ref:`ignoreconfig`.
+
+Configuration Issues
+^^^^^^^^^^^^^^^^^^^^
+
+If you're getting ether unexpected behaviour with your config, or errors
+because config values haven't been set correctly, it's often due to config
+file discovery (i.e. whether SQLFluff can find your config file, and what
+order it's combining config files).
+
+For a more general guide to this topic see :ref:`setting_config`.
+
+To help troubleshoot issues, if you run ``sqlfluff`` with a more verbose
+logging setting (e.g. ``sqlfluff lint /my/model.sql -v``, or ``-vv``, or
+``-vvvvvv``) you'll get a readout of the root config that SQLFluff is using.
+This can help debug which values are being used.
+
+2. Isolating SQLFluff
+---------------------
+
+If you're still getting strange errors, then the next most useful thing you
+can do, both to help narrow down the cause, but also to assist with fixing
+a bug if you have found one, is to isolate SQLFluff from any other tools
+you're using in parallel:
+
+1. If you're using SQLFluff with the :ref:`dbt_templater`, then try and
+   recreate the error with the :ref:`jinja_templater` to remove the influence
+   of ``dbt`` and any database connection related issues.
+
+2. If you're getting an error in a remote CI suite (for example on GitHub
+   actions, or a server like Jenkins), try and recreate the issue locally
+   on your machine using the same tools.
+
+3. If you're :ref:`using-pre-commit`, :ref:`diff-quality` or the
+   `VSCode extension`_ try to recreate the issue by running the SQLFluff
+   :ref:`cliref` directly. Often this can make debugging significantly
+   easier because some of these tools hide some of the error messages
+   which SQLFluff gives the user to help debugging errors.
+
+.. _`VSCode extension`: https://github.com/sqlfluff/vscode-sqlfluff
+
+3. Minimise the Query
+---------------------
+
+Often SQL scripts can get very long, and if you're getting an error on a very
+long script, then it can be extremely difficult to work out what the issue is.
+To assist with this we recommend iteratively cutting down the file (or
+alternatively, iteratively building a file back up) until you have the smallest
+file which still exhibits the issue. Often after this step, the issue can
+become obvious.
+
+1. If your file has multiple statements in it (i.e. statements separated
+   by ``;``), then remove ones until SQLFluff no longer shows the issue. When
+   you get to that point, add the offending one back in and remove all the
+   others.
+
+2. Simplify individual statements. For example in a ``SELECT`` statement, if
+   you suspect the issue is coming from a particular column, then remove the
+   others, or remove CTEs, until you've got the simplest query which still
+   shows the issue.

docs/source/indentation.rst

@@ -1,240 +0,0 @@
-.. _indentref:
-
-Let's talk about indentation
-============================
-
-If there is one part of building a linter that is going to be controversial
-it's going to be **indentation** (closely followed by **cApiTaLiSaTiOn** 😁).
-
-*SQLFluff* aims to be *opinionated* here, but also *configurable* (see
-:ref:`indentconfig`). The tool will have a default viewpoint and will aim
-to have views on all of the important aspects of SQL layout, but if you
-(or your organisation) don't like those views then we aim to allow enough
-configuration that you can lint in line with your views, and still use
-*SQLFluff*. For more information on how to configure rules to your own
-viewpoint see :ref:`config`.
-
-So, without further ado, here are the principles we think apply to indentation:
-
-1. **For Keywords within a statement, the first root keyword of each line should be
-   aligned.** For :code:`SELECT` statements, this means that :code:`SELECT`,
-   :code:`FROM`, :code:`WHERE`, :code:`GROUP`, :code:`ORDER`, :code:`HAVING`
-   and :code:`LIMIT`, should all have the same indent. Occasionally, it's
-   actually more legible to have one-line or more compressed statements,
-   and so additionally, if two (or more) of these keywords are on *the same*
-   line, then the second (and any further) keywords won't raise a violation,
-   provided that the *first* was correctly aligned.
-
-   * This same logic applies to keywords within subsections, but the
-     likelihood of them being on the same line to start is higher. one
-     example of where this might occur regularly is within aggregate
-     functions.
-
-     .. code-block:: sql
-
-         SELECT
-            col_a,
-            col_b,
-            COUNT(*) as num,
-            SUM(num) OVER (
-               PARTITION BY col_a
-               ORDER BY col_b
-            ) as an_aggregate_function
-         FROM tbl_a
-         GROUP BY 1, 2
-
-     Note that :code:`PARTITION` and :code:`ORDER` are both aligned on
-     the same line. This also follows the rules around brackets described
-     below.
-
-2. **Line Length**. Long lines are hard to read and many SQL guidelines
-   include a line length restriction. This is (of course) configurable, but
-   the default is 80 characters (in line with the `dbt Labs SQL style guide`_.)
-
-3. **Bracket behaviour**. For brackets there are three accepted ways:
-
-   a. *Inline brackets*. Bracket expressions that start and end on the same line are
-      fine (providing we don't fall foul of the line length rules above).
-
-      .. code-block:: sql
-
-         SELECT GREATEST(1, 6) AS col1 FROM my_table
-
-   b. *Brackets with immediate linebreak*. If brackets are followed by an immediate
-      line break (or at least with no other non-code elements after them on that
-      line), then the following line should be indented +1 relative to the
-      indent of the previous line. All elements of the bracketed block should
-      be at this level of indent or deeper. The *closing* bracket of this block
-      should have the same indent as the first element of the line containing
-      the opening bracket.
-
-      .. code-block:: sql
-
-         SELECT GREATEST(
-             1, 3, 7,
-             6, 8, 9
-         ) AS col1
-         FROM my_table
-
-   c. *Brackets with delayed linebreak*. If brackets are followed by content,
-      and then a linebreak *before* the closing bracket, then we assume a
-      *hanging* indent, where the following items of content should have the
-      same indent as the first item of content. In this case, the *closing*
-      bracket should come after the final element *on the same line*.
-
-      .. code-block:: sql
-
-         SELECT GREATEST(1, 6, 8,
-                         6, 7) AS col1
-         FROM my_table
-
-4. **Comments** are dealt with differently, depending on whether they're
-   *block* comments (:code:`/* like this */`), which might optionally
-   include newlines, or *inline* comments (:code:`-- like this`) which
-   are necessarily only on one line.
-
-   a. *Block comments* cannot share a line with any code elements (so
-      in effect they must start on their own new line), they cannot be
-      followed by any code elements on the same line (and so in effect
-      must be followed by a newline, if we are to avoid trailing
-      whitespace). None of the lines within the block comment may have
-      an indent less than the first line of the block comment (although
-      additional indentation within a comment is allowed), and that first
-      line should be aligned with the first code element *following*
-      the block comment.
-
-      .. code-block:: sql
-
-         SELECT
-            /* This is a block comment starting on a new line
-            which contains a newline (continuing with at least
-            the same indent.
-               - potentially containing greater indents
-               - having no other code following it in the same line
-               - and aligned with the line of code following it */
-            this_column as what_we_align_the_column_to
-         FROM my_table
-
-   b. *Inline comments* can be on the same line as other code, but are
-      subject to the same line-length restrictions. If they don't fit
-      on the same line (or if it just looks nicer) they can also be
-      the only element on a line. In this latter case they should be
-      aligned with the first code element *following* the comment.
-
-      .. code-block:: sql
-
-         SELECT
-            -- This is fine
-            this_column as what_we_align_to,
-            another_column as something_short,  -- Is ok
-            case
-               -- This is aligned correctly with below
-               when indented then take_care
-               else try_harder
-            end as the_general_guidance
-         -- Even here we align with the line below
-         FROM my_table
-
-      .. note::
-
-         When fixing issues with comment indentation, SQLFluff
-         will attempt to keep comments in their original position
-         but if line length concerns make this difficult, it will
-         either abort the fix, or move *same line* comments up and
-         *before* the line they are currently on. This is in line
-         with the assumption that comments on their own line refer
-         to the elements of code which they come *before*, not *after*.
-
-
-.. _indentconfig:
-
-Configuring Indentation
-^^^^^^^^^^^^^^^^^^^^^^^
-
-How indentation is linted is controlled in the rules, but what indentation
-is expected to be present is controlled by the parser, and therefore
-configured separately. One of the key areas for this is the indentation
-of the :code:`JOIN` expression.
-
-Semantically, a :code:`JOIN` expression is part of the :code:`FROM` expression
-and therefore would be expected to be indented. However according to many
-of the most common SQL style guides (including the `dbt Labs SQL style guide`_
-and the `Mozilla SQL style guide`_) the :code:`JOIN` keyword is expected to at
-the same indent as the :code:`FROM` keyword. By default, *SQLFluff* sides with
-the current consensus, which is to *not* indent the :code:`JOIN` keyword,
-however this is one element which is configurable.
-
-By setting values in the :code:`sqlfluff:indentation` section of your config
-file you can control how this is parsed, for example you may work with an
-indentation similar to that of `Baron Schwartz`_.
-
-By setting your config file to:
-
-.. code-block:: cfg
-
-   [sqlfluff:indentation]
-   indented_joins = True
-
-Then the expected indentation will be:
-
-.. code-block:: sql
-
-   SELECT
-      a, b, c
-   FROM my_table
-      JOIN another_table
-         USING(a)
-
-However if no value for :code:`indented_joins` is set, or if it is set to
-:code:`False` then then following indentation will be expected:
-
-.. code-block:: sql
-
-   SELECT
-      a, b, c
-   FROM my_table
-   JOIN another_table
-      USING(a)
-
-There is a similar :code:`indented_using_on` config (defaulted to :code:`True`)
-which can be set to :code:`False` to prevent the :code:`using` clause from
-being indented, in which case above SQL would become:
-
-.. code-block:: sql
-
-   SELECT
-      a, b, c
-   FROM my_table
-   JOIN another_table
-   USING(a)
-
-There is also a similar :code:`indented_ctes` config (defaulted to
-:code:`False`) which can be set to :code:`True` to enforce CTEs to be
-indented within the :code:`WITH` clause:
-
-.. code-block:: sql
-
-   WITH
-      some_cte AS (
-         SELECT 1 FROM table1
-      ),
-
-      some_other_cte AS (
-         SELECT 1 FROM table1
-      )
-
-   SELECT 1 FROM some_cte
-
-Note that using :code:`indented_ctes` may clash with `Rule L018`_ (`"WITH
-clause closing bracket should be aligned with WITH keyword."``), so if using
-this option you will likely want to disable that rule.
-
-By default, *SQLFluff* aims to follow the indentation most common approach
-to indentation. However, if you have other versions of indentation which are
-supported by published style guides, then please submit an issue on GitHub
-to have that variation supported by *SQLFluff*.
-
-.. _`dbt Labs SQL style guide`: https://github.com/dbt-labs/corp/blob/master/dbt_style_guide.md
-.. _`Mozilla SQL style guide`: https://docs.telemetry.mozilla.org/concepts/sql_style.html#joins
-.. _`Baron Schwartz`: https://www.xaprb.com/blog/2006/04/26/sql-coding-standards/
-.. _`Rule L018`: ./rules.html#sqlfluff.core.rules.Rule_L018

docs/source/index.rst

@@ -6,36 +6,20 @@ you're working with? Fluff is an extensible and modular linter designed
 to help you write good SQL and catch errors and bad SQL before it hits
 your database.
 
-.. note::
-
-    **SQLFluff** is still in an open Beta phase - expect potentially
-    non-backward compatible api changes to happen with releases
-    (particularly when major or minor version numbers increase).
-
-Notable changes:
-
-* **0.1.x** involved a major re-write of the parser, completely changing
-  the behaviour of the tool with respect to complex parsing.
-* **0.2.x** added templating support and a big restructure of rules
-  and changed how users might interact with SQLFluff on templated code.
-* **0.3.x** drops support for python 2.7 and 3.4, and also reworks the
-  handling of indentation linting in a potentially not backward
-  compatible way.
-* **0.4.x** dropped python 3.5, added the dbt templater, source mapping and
-  also introduced the python API
-* **0.5.x** introduced some breaking changes to the API.
-* **0.6.x** introduced parallel processing, which necessitated a big re-write
-  of several innards
-* **0.7.x** extracted the dbt templater to a separate plugin and removed the
-  ``exasol_fs`` dialect (now merged in with the main ``exasol``)
-* **0.8.x** an improvement to the performance of the parser, a rebuild of the
-  Jinja Templater, and a progress bar for the CLI
-* **0.9.x** refinement of the Simple API, dbt 1.0.0 compatibility,
-  and the official SQLFluff Docker image.
-* **0.10.x** removed support for older dbt versions < 0.20 and stopped ``fix``
-  attempting to fix unparsable SQL.
+Notable releases:
 
+* **1.0.x**: First *stable* release, no major changes to take advantage of a
+  point of relative stability.
+* **2.0.x**: Recode of rules, whitespace fixing consolidation,
+  :code:`sqlfluff format` and removal of support for dbt versions pre `1.1`.
+  Note, that this release brings with it some breaking changes to rule coding
+  and configuration, see :ref:`upgrading_2_0`.
+* **3.0.x**: :code:`sqlfluff fix` now defaults to *not* asking for confirmation
+  and the `--force` option was removed. Richer information returned by the
+  :code:`sqlfluff lint` command (although in a different structure to previous
+  versions). See :ref:`upgrading_3_0`.
 
+For more detail on other releases, see our :ref:`releasenotes`.
 
 Want to see where and how people are using SQLFluff in their projects?
 Head over to :ref:`inthewildref` for inspiration.
@@ -51,11 +35,23 @@ have python or pip already installed see :ref:`gettingstartedref`.
 
     $ pip install sqlfluff
     $ echo "  SELECT a  +  b FROM tbl;  " > test.sql
-    $ sqlfluff lint test.sql
+    $ sqlfluff lint test.sql --dialect ansi
     == [test.sql] FAIL
-    L:   1 | P:   1 | L003 | Single indentation uses a number of spaces not a multiple of 4
-    L:   1 | P:  14 | L006 | Operators should be surrounded by a single space unless at the start/end of a line
-    L:   1 | P:  27 | L001 | Unnecessary trailing whitespace
+    L:   1 | P:   1 | LT01 | Expected only single space before 'SELECT' keyword.
+                           | Found '  '. [layout.spacing]
+    L:   1 | P:   1 | LT02 | First line should not be indented.
+                           | [layout.indent]
+    L:   1 | P:   1 | LT13 | Files must not begin with newlines or whitespace.
+                           | [layout.start_of_file]
+    L:   1 | P:  11 | LT01 | Expected only single space before binary operator '+'.
+                           | Found '  '. [layout.spacing]
+    L:   1 | P:  14 | LT01 | Expected only single space before naked identifier.
+                           | Found '  '. [layout.spacing]
+    L:   1 | P:  27 | LT01 | Unnecessary trailing whitespace at end of file.
+                           | [layout.spacing]
+    L:   1 | P:  27 | LT12 | Files must end with a single trailing newline.
+                           | [layout.end_of_file]
+    All Finished 📜 🎉!
 
 Contents
 ^^^^^^^^
@@ -65,19 +61,11 @@ Contents
    :caption: Documentation for SQLFluff:
 
    gettingstarted
-   realworld
-   vision
-   teamrollout
-   indentation
-   rules
-   dialects
-   production
-   configuration
-   architecture
-   cli
-   api
-   developingrules
-   developingplugins
+   why_sqlfluff
+   guides/index
+   configuration/index
+   production/index
+   reference/index
    inthewild
    jointhecommunity
 

docs/source/inthewild.rst

@@ -45,7 +45,7 @@ Just add a section below by raising a PR on GitHub by
   to this project. In particular our annual `Web Almanac <https://almanac.httparchive.org>`_
   attracts hundreds of volunteers to help analyse our BigQuery dataset and
   being able automatically lint Pull Requests through GitHub Actions is a
-  fantastic way to help us maintain our growing repositary of
+  fantastic way to help us maintain our growing repository of
   `over a thousand queries <https://github.com/HTTPArchive/almanac.httparchive.org/tree/main/sql>`_.
 - `Brooklyn Data Co <https://www.brooklyndata.co>`_ has a `dbt_artifacts <https://github.com/brooklyn-data/dbt_artifacts>`_
   dbt package from which runs SQLFluff in CI to lint pull requests
@@ -54,3 +54,26 @@ Just add a section below by raising a PR on GitHub by
   contributed by Greg Clunies, with annotations on pull requests to make it
   easy for contributors to see where their SQL has failed any rules. See an
   `example pull request with SQLFluff annotations <https://github.com/brooklyn-data/dbt_artifacts/pull/74/files>`_.
+- `Markerr <https://www.markerr.com>`_ has tightly integrated SQLFluff into our
+  CI/CD process for data model changes and process improvements. Since adopting
+  SQLFluff across the organization, the clarity of our SQL code has risen
+  dramatically, freeing up review time to focus on deeper data and
+  process-specific questions.
+- `Symend <https://www.symend.com>`_ has a microservices platform supporting
+  our SaaS product. We use SQLFLuff in the CI/CD process of several of our
+  data-oriented microservices. Among other things, it validates our database
+  migration scripts, deployed using
+  `schemachange <https://github.com/Snowflake-Labs/schemachange>`_ and we have
+  near-term plans to implement it for our `dbt`_ projects.
+- At `CarePay <https://www.carepay.com>`_ we use SQLFLuff to lint and fix all
+  our dbt models as well as several other SQL heavy projects. Locally we use
+  SQLFluff with pre-commit and have also integrated it into our CI/CD
+  pipelines.
+- Core Analytics Team from `Typeform <https://www.typeform.com/>`_ and
+  `videoask <https://www.videoask.com/>`_ uses SQLFluff in the production
+  `dbt <http://www.getdbt.com/>`_ project for building our datawarehouse
+  layer for both products:
+
+  - We use it locally in our day to day work, helping us to write cleaner code.
+  - We added SQLFluff to our CI processes, so during a PR we can check that any
+    new or modified sql file has a consistent and easy-to-read format.

docs/source/jointhecommunity.rst

@@ -1,12 +1,12 @@
-.. _jointhecommunity:
+.. _sqlfluff_slack:
 
 SQLFluff Slack
-====================
+==============
 
-We have a fast-growing `community on Slack <https://join.slack.com/t/sqlfluff/shared_invite/zt-o1f4x0e8-pZzarAIlQmKj_6ZwD16w0g>`_, come and join us!
+We have a fast-growing `community on Slack <https://join.slack.com/t/sqlfluff/shared_invite/zt-2qtu36kdt-OS4iONPbQ3aCz2DIbYJdWg>`_, come and join us!
 
 
 SQLFluff on Twitter
 ====================
 
-Follow us On Twitter `@SQLFluff <https://twitter.com/SQLFluff>`_ for annoncements and other related posts.
+Follow us On Twitter `@SQLFluff <https://twitter.com/SQLFluff>`_ for announcements and other related posts.

docs/source/production.rst

@@ -1,173 +0,0 @@
-Production Usage
-================
-
-SQLFluff is designed to be used both as a utility for developers but also to
-be part of `CI/CD`_ pipelines.
-
-Using SQLFluff on a whole sql codebase
---------------------------------------
-
-The `exit code`_ provided by SQLFluff when run as a command line utility is
-designed to assist usefulness in deployment pipelines. If no violations
-are found then the `exit code`_ will be 0. If violations are found then
-a non-zero code will be returned which can be interrogated to find out
-more.
-
-- At the moment all error states related to linting return *65*.
-- An error as a result of a SQLFluff internal error will return *1*.
-
-.. _`CI/CD`: https://en.wikipedia.org/wiki/Continuous_integration
-.. _`exit code`: https://shapeshed.com/unix-exit-codes/
-
-.. _diff-quality:
-
-Using SQLFluff on changes using `diff-quality`
-----------------------------------------------
-
-For projects with large amounts of (potentially imperfect) SQL code, the full
-SQLFluff output could be very large, which can be distracting -- perhaps the CI
-build for a one-line SQL change shouldn't encourage the developer to fix lots
-of unrelated quality issues.
-
-To support this use case, SQLFluff integrates with a quality checking tool
-called `diff-quality`. By running SQLFluff using `diff-quality` (rather than
-running it directly), you can limit the the output to the new or modified SQL
-in the branch (aka pull request or PR) containing the proposed changes.
-
-Currently, ``diff-quality`` requires that you are using ``git`` for version
-control.
-
-NOTE: Installing SQLFluff automatically installs the `diff_cover` package that
-provides the `diff-quality` tool.
-
-Adding `diff-quality` to your builds
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-In your CI build script:
-
-1. Set the current working directory to the ``git`` repository containing the
-SQL code to be checked.
-
-2. Run `diff-quality`, specifying SQLFluff as the underlying tool:
-
-.. code-block:: text
-
-    $ diff-quality --violations sqlfluff
-
-The output will look something like:
-
-.. code-block:: text
-
-    -------------
-    Diff Quality
-    Quality Report: sqlfluff
-    Diff: origin/master...HEAD, staged and unstaged changes
-    -------------
-    sql/audience_size_queries/constraints/_postcondition_check_gdpr_compliance.sql (0.0%):
-    sql/audience_size_queries/constraints/_postcondition_check_gdpr_compliance.sql:5: Unquoted Identifiers must be consistently upper case.
-    -------------
-    Total:   1 line
-    Violations: 1 line
-    % Quality: 0%
-    -------------
-
-These messages are basically the same as those provided directly by SQLFluff,
-although the format is a little different. Note that `diff-quality` only lists
-the line _numbers_, not the character position. If you need the character
-position, you will need to run SQLFluff directly.
-
-For more information on `diff-quality`, see the
-`documentation <https://diff-cover.readthedocs.io/en/latest/>`_. It covers topics
-such as:
-
-* Generating HTML reports
-* Controlling which branch to compare against (i.e. to determine new/changed
-  lines). The default is `origin/master`.
-* Configuring `diff-quality` to return an error code if the quality is too low
-* Troubleshooting
-
-Using `pre-commit`_
-^^^^^^^^^^^^^^^^^^^
-
-`pre-commit`_ is a framework to manage git "hooks"
-triggered right before a commit is made.
-
-A `git hook`_ is a git feature to "fire off custom scripts"
-when specific actions occur.
-
-Using `pre-commit`_ with SQLFluff is a good way
-to provide automated linting to SQL developers.
-
-With `pre-commit`_, you also get the benefit of
-only linting/fixing the files that changed.
-
-SQLFluff comes with two `pre-commit`_ hooks:
-
-* sqlfluff-lint: returns linting errors.
-* sqlfluff-fix: attempts to fix rule violations.
-
-.. warning::
-   For safety reasons, ``sqlfluff-fix`` by default will not make any fixes in
-   files that had templating or parse errors, even if those errors were ignored
-   using ``noqa`` or `--ignore``.
-
-   Although it is not advised, you *can* tell SQLFluff to try and fix
-   these files by overriding the ``fix_even_unparsable`` setting
-   in ``.sqlfluff`` config file or using the ``sqlfluff fix --FIX-EVEN-UNPARSABLE``
-   command line option.
-
-   *Overriding this behavior may break your SQL. If you use this override,
-   always be sure to review any fixes applied to files with templating or parse
-   errors to verify they are okay.*
-
-You should create a file named `.pre-commit-config.yaml`
-at the root of your git project, which should look
-like this:
-
-.. code-block:: yaml
-
-  repos:
-  - repo: https://github.com/sqlfluff/sqlfluff
-    rev: |release|
-    hooks:
-      - id: sqlfluff-lint
-        # For dbt projects, this installs the dbt "extras".
-        # You will need to select the relevant dbt adapter for your dialect
-        # (https://docs.getdbt.com/docs/available-adapters):
-        # additional_dependencies: ['<dbt-adapter>', 'sqlfluff-templater-dbt']
-      - id: sqlfluff-fix
-        # Arbitrary arguments to show an example
-        # args: [--rules, "L003,L014"]
-        # additional_dependencies: ['<dbt-adapter>', 'sqlfluff-templater-dbt']
-
-When trying to use the `dbt templater`_, uncomment the
-``additional_dependencies`` to install the extras.
-This is equivalent to running ``pip install <dbt-adapter> sqlfluff-templater-dbt``.
-
-You can specify the verion of ``dbt-adapter`` used in `pre-commit`_,
-for example:
-
-.. code-block:: yaml
-
-   additional_dependencies : ['dbt-bigquery==1.0.0', 'sqlfluff-templater-dbt']
-
-See the list of available `dbt-adapters`_.
-
-Note that you can pass the same arguments available
-through the CLI using ``args:``.
-
-Using `GitHub Actions`_ to Annotate PRs
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-When `sqlfluff lint` is run with the `--format github-annotation` option, it
-produces output compatible with this `action <https://github.com/yuzutech/annotations-action>`_.
-You can use this to annotate the SQL in `GitHub pull requests`.
-
-For more information and examples on using SQLFluff in GitHub Actions, see the
-`sqlfluff-github-actions repository <https://github.com/sqlfluff/sqlfluff-github-actions>`_.
-
-.. _`pre-commit`: https://pre-commit.com/
-.. _`git hook`: https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks
-.. _`dbt templater`: `dbt-project-configuration`
-.. _`GitHub Actions`: https://github.com/features/actions
-.. _`GitHub pull requests`: https://docs.github.com/en/github/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-pull-requests
-.. _`dbt-adapters`: https://docs.getdbt.com/docs/available-adapters

docs/source/production/cli_use.rst

@@ -0,0 +1,25 @@
+Using SQLFluff directly as a CLI application
+--------------------------------------------
+
+The :ref:`SQLFluff CLI application <cliref>` is a python application which
+means if depends on your host python environment
+(see :ref:`installingsqlfluff`).
+
+The `exit code`_ provided by SQLFluff when run as a command line utility is
+designed to assist usefulness in deployment pipelines. If no violations
+are found then the `exit code`_ will be 0. If violations are found then
+a non-zero code will be returned which can be interrogated to find out
+more.
+
+- An error code of ``0`` means *operation success*, *no issues found*.
+- An error code of ``1`` means *operation success*, *issues found*. For
+  example this might mean that a linting issue was found, or that one file
+  could not be parsed.
+- An error code of ``2`` means an error occurred and the operation could
+  not be completed. For example a configuration issue or an internal error
+  within SQLFluff.
+
+For details of what commands and options are available in the CLI see the
+:ref:`cliref`.
+
+.. _`exit code`: https://shapeshed.com/unix-exit-codes/

docs/source/production/diff_quality.rst

@@ -0,0 +1,98 @@
+.. _diff-quality:
+
+Using SQLFluff on changes using ``diff-quality``
+================================================
+
+For projects with large amounts of (potentially imperfect) SQL code, the full
+SQLFluff output could be very large, which can be distracting -- perhaps the CI
+build for a one-line SQL change shouldn't encourage the developer to fix lots
+of unrelated quality issues.
+
+To support this use case, SQLFluff integrates with a quality checking tool
+called ``diff-quality``. By running SQLFluff using ``diff-quality`` (rather
+than running it directly), you can limit the the output to the new or modified
+SQL in the branch (aka pull request or PR) containing the proposed changes.
+
+Currently, ``diff-quality`` requires that you are using ``git`` for version
+control.
+
+NOTE: Installing SQLFluff automatically installs the ``diff_cover`` package
+that provides the ``diff-quality`` tool.
+
+Adding ``diff-quality`` to your builds
+--------------------------------------
+
+In your CI build script:
+
+1. Set the current working directory to the ``git`` repository containing the
+   SQL code to be checked.
+
+2. Run ``diff-quality``, specifying SQLFluff as the underlying tool:
+
+.. code-block:: text
+
+    $ diff-quality --violations sqlfluff
+
+The output will look something like:
+
+.. code-block:: text
+
+    -------------
+    Diff Quality
+    Quality Report: sqlfluff
+    Diff: origin/master...HEAD, staged and unstaged changes
+    -------------
+    sql/audience_size_queries/constraints/_postcondition_check_gdpr_compliance.sql (0.0%):
+    sql/audience_size_queries/constraints/_postcondition_check_gdpr_compliance.sql:5: Unquoted Identifiers must be consistently upper case.
+    -------------
+    Total:   1 line
+    Violations: 1 line
+    % Quality: 0%
+    -------------
+
+These messages are basically the same as those provided directly by SQLFluff,
+although the format is a little different. Note that ``diff-quality`` only lists
+the line _numbers_, not the character position. If you need the character
+position, you will need to run SQLFluff directly.
+
+.. note::
+   When using ``diff-quality`` with ``.sqlfluff`` :ref:`config-files`, and
+   especially together with the :ref:`dbt_templater`, it can be really easy
+   to run into issues with file discovery. There are a few steps you can
+   take to make it much less likely that this will happen:
+
+   1. ``diff-quality`` needs to be run from the root of your ``git``
+      repository (so that it can find the ``git`` metadata).
+
+   2. SQLFluff works best if the bulk of the configuration is done from a
+      single ``.sqlfluff`` file, which should be in the root of your
+      ``git`` repository.
+
+   3. If using :ref:`dbt_templater`, then either place your ``dbt_project.yml``
+      file in the same root folder, or if you put it in a subfolder, then
+      only invoke ``diff-quality`` and ``sqlfluff`` from the root and define
+      the subfolder that the ``dbt`` project lives in using the ``.sqlfluff``
+      config file.
+
+   By aligning the paths of all three, you should be able to achieve a
+   robust setup. If each is rooted in different paths if can be very
+   difficult to achieve the same result, and the resulting behaviour
+   can be difficult to debug.
+
+   To debug any issues relating to this setup, we recommend occasionally
+   running ``sqlfluff`` directly using the main cli (i.e. calling
+   :code:`sqlfluff lint my/project/path`) and check whether that route
+   gives you the results you expect. ``diff-quality`` should behave as
+   though it's calling the SQLFluff CLI *from the same path that you*
+   *invoke* ``diff-quality``.
+
+For more information on ``diff-quality`` and the ``diff_cover`` package, see the
+`documentation <https://github.com/Bachmann1234/diff_cover>`_ on their github
+repository. It covers topics such as:
+
+* Generating HTML reports
+* Controlling which branch to compare against (i.e. to determine new/changed
+  lines). The default is ``origin/main``.
+* Configuring ``diff-quality`` to return an error code if the quality is
+  too low.
+* Troubleshooting

docs/source/production/github_actions.rst

@@ -0,0 +1,30 @@
+Using `GitHub Actions`_ to Annotate PRs
+=======================================
+
+There are two way to utilize SQLFluff to annotate Github PRs.
+
+1. When :code:`sqlfluff lint` is run with the :code:`--format github-annotation-native`
+   option, it produces output formatted as `Github workflow commands`_ which
+   are converted into pull request annotations by Github.
+
+2. When :code:`sqlfluff lint` is run with the :code:`--format github-annotation`
+   option, it produces output compatible with this `action from yuzutech`_.
+   Which uses Github API to annotate the SQL in `GitHub pull requests`.
+
+.. warning::
+   At present (December 2023), limitations put in place by Github mean that only the
+   first 10 annotations will be displayed if the first option (using
+   :code:`github-annotation-native`) is used. This is a not something that SQLFluff
+   can control itself and so we currently recommend using the the second option
+   above and the `action from yuzutech`_.
+
+   There is an `open feature request <https://github.com/orgs/community/discussions/68471>`_
+   for GitHub Actions which you can track to follow this issue.
+
+For more information and examples on using SQLFluff in GitHub Actions, see the
+`sqlfluff-github-actions repository <https://github.com/sqlfluff/sqlfluff-github-actions>`_.
+
+.. _`GitHub Actions`: https://github.com/features/actions
+.. _`GitHub pull requests`: https://docs.github.com/en/github/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-pull-requests
+.. _`Github workflow commands`: https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-a-notice-message
+.. _`action from yuzutech`: https://github.com/yuzutech/annotations-action

docs/source/production/index.rst

@@ -0,0 +1,18 @@
+.. _production-use:
+
+Production Usage & Security
+===========================
+
+SQLFluff is designed to be used both as a utility for developers but also to
+be part of `CI/CD`_ pipelines.
+
+.. _`CI/CD`: https://en.wikipedia.org/wiki/Continuous_integration
+
+.. toctree::
+   :maxdepth: 2
+
+   security
+   cli_use
+   diff_quality
+   pre_commit
+   github_actions

docs/source/production/pre_commit.rst

@@ -0,0 +1,99 @@
+.. _using-pre-commit:
+
+Using :code:`pre-commit`
+========================
+
+`pre-commit`_ is a framework to manage git "hooks"
+triggered right before a commit is made.
+
+A `git hook`_ is a git feature to "fire off custom scripts"
+when specific actions occur.
+
+Using `pre-commit`_ with SQLFluff is a good way
+to provide automated linting to SQL developers.
+
+With `pre-commit`_, you also get the benefit of
+only linting/fixing the files that changed.
+
+SQLFluff comes with two `pre-commit`_ hooks:
+
+* ``sqlfluff-lint``: returns linting errors.
+* ``sqlfluff-fix``: attempts to fix rule violations.
+
+.. warning::
+   For safety reasons, ``sqlfluff-fix`` by default will not make any fixes in
+   files that had templating or parse errors, even if those errors were ignored
+   using ``noqa`` or `--ignore``.
+
+   Although it is not advised, you *can* tell SQLFluff to try and fix
+   these files by overriding the ``fix_even_unparsable`` setting
+   in ``.sqlfluff`` config file or using the ``sqlfluff fix --FIX-EVEN-UNPARSABLE``
+   command line option.
+
+   *Overriding this behavior may break your SQL. If you use this override,
+   always be sure to review any fixes applied to files with templating or parse
+   errors to verify they are okay.*
+
+You should create a file named ``.pre-commit-config.yaml``
+at the root of your git project, which should look
+like this:
+
+.. code-block:: yaml
+
+  repos:
+  - repo: https://github.com/sqlfluff/sqlfluff
+    rev: |release|
+    hooks:
+      - id: sqlfluff-lint
+        # For dbt projects, this installs the dbt "extras".
+        # You will need to select the relevant dbt adapter for your dialect
+        # (https://docs.getdbt.com/docs/available-adapters):
+        # additional_dependencies: ['<dbt-adapter>', 'sqlfluff-templater-dbt']
+      - id: sqlfluff-fix
+        # Arbitrary arguments to show an example
+        # args: [--rules, "LT02,CP02"]
+        # additional_dependencies: ['<dbt-adapter>', 'sqlfluff-templater-dbt']
+
+When trying to use the `dbt templater`_, uncomment the
+``additional_dependencies`` to install the extras.
+This is equivalent to running ``pip install <dbt-adapter> sqlfluff-templater-dbt``.
+
+You can specify the version of ``dbt-adapter`` used in `pre-commit`_,
+for example:
+
+.. code-block:: yaml
+
+   additional_dependencies : ['dbt-bigquery==1.0.0', 'sqlfluff-templater-dbt']
+
+See the list of available `dbt-adapters`_.
+
+Note that you can pass the same arguments available
+through the CLI using ``args:``.
+
+.. _`pre-commit`: https://pre-commit.com/
+.. _`git hook`: https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks
+.. _`dbt templater`: `dbt-project-configuration`
+.. _`dbt-adapters`: https://docs.getdbt.com/docs/available-adapters
+
+Ignoring files while using :code:`pre-commit`
+---------------------------------------------
+
+Under the hood, `pre-commit`_ works by passing specific files to *SQLFluff*.
+For example, if the only two files that are modified in your commit are
+:code:`file_a.sql` and :code:`file_b.sql`, then the command which is called
+in the background is :code:`sqlfluff lint file_a.sql file_b.sql`. While this
+is efficient, it does produce some unwanted noise when also
+using :ref:`sqlfluffignore`. This is because *SQLFluff* is designed to allow
+users to override an *ignore* configuration by passing the name of the file
+directly. This makes a lot of sense in a CLI context, but less so in the context
+of being invoked by `pre-commit`_.
+
+To avoid noisy logs when using both `pre-commit`_ and :ref:`sqlfluffignore`,
+we recommend also setting the :code:`exclude` argument in your
+:code:`.pre-commit-config.yaml` file (either the `top level config`_ or the
+`hook specific config`_). This will prevent files matching the given pattern
+being passed to *SQLFluff* and so silence any warnings about the
+:ref:`sqlfluffignore` being overridden.
+
+.. _`top level config`: https://pre-commit.com/#top_level-exclude
+.. _`hook specific config`: https://pre-commit.com/#config-exclude

docs/source/production/security.rst

@@ -0,0 +1,56 @@
+.. _security:
+
+Security Considerations
+=======================
+
+A full list of `Security Advisories is available on GitHub <https://github.com/sqlfluff/sqlfluff/security/advisories>`_.
+
+Given the context of how SQLFluff is designed to be used, there are three
+different tiers of access which users may have access to manipulate how the
+tool functions in a secure environment.
+
+#. *Users may have edit access to the SQL code which is being linted*. While
+   SQLFluff does not execute the SQL itself, in the process of the
+   :ref:`templating step <templater>` (in particular via jinja or dbt),
+   certain macros may have the ability to execute arbitrary SQL code (e.g.
+   the `dbt run_query macro`_). For the Jinja templater, SQLFluff uses the
+   `Jinja2 SandboxedEnvironment`_ to limit the execution on unsafe code. When
+   looking to further secure this situation, see below for ways to limit the
+   ability of users to import other libraries.
+
+#. *Users may have edit access to the SQLFluff :ref:`config-files`*. In some
+   (perhaps, many) environments, the users who can edit SQL files may also
+   be able to access and edit the :ref:`config-files`. It's important to note
+   that because of :ref:`in_file_config`, that users who can edit SQL files
+   which are designed to be linted, will also have access to the vast majority
+   of any configuration options available in :ref:`config-files`. This means
+   that there is minimal additional protection from restricting access to
+   :ref:`config-files` for users who already have access to edit the linting
+   target files (as described above).
+
+#. *Users may have access to change how SQLFluff is invoked*. SQLFluff can
+   be invoked either as a command line too or via the python API. Typically
+   the method is fixed for a given application. When thinking about how to
+   restrict the ability of users to call insecure code, SQLFluff aims to
+   provide options at the point of invocation. In particular, as described
+   above, the primary risk vector for SQLFluff is the macro environment
+   as described in :ref:`templateconfig`. To restrict users being able to
+   bring arbitrary python methods into sqlfluff via the ``library_path``
+   configuration value (see :ref:`jinja_library_templating`), we recommend
+   that for secure environments you override this config value either by
+   providing an ``override`` option to the :class:`FluffConfig` object if
+   using the Python API or via the ``--library-path`` CLI option:
+
+   To disable this option entirely via the CLI:
+
+   .. code-block:: bash
+
+      $ sqlfluff lint my_path --library-path none
+
+   To disable this option entirely via the python API:
+
+   .. literalinclude:: ../../../examples/04_config_overrides.py
+      :language: python
+
+.. _`Jinja2 SandboxedEnvironment`: https://jinja.palletsprojects.com/en/3.0.x/sandbox/#jinja2.sandbox.SandboxedEnvironment
+.. _`dbt run_query macro`: https://docs.getdbt.com/reference/dbt-jinja-functions/run_query

docs/source/realworld.rst

@@ -1,101 +0,0 @@
-.. _realworldref:
-
-SQL in the Wild
-===============
-
-SQL has been around for a long time, as a language for communicating
-with databases, like a communication protocol. More recently with the
-rise of *data* as a business function, or a domain in its own right
-SQL has also become an invaluable tool for defining the *structure* of
-data and analysis - not just as a one off but as a form of
-`infrastructure as code`_.
-
-As *analytics* transitions from a profession of people doing one-offs,
-and moves to building stable and reusable pieces of analytics, more and
-more principles from software engineering are moving in the analytics
-space. One of the best articulations of this is written in the
-`viewpoint section of the docs for the open-source tool dbt`_. Two of
-the principles mentioned in that article are `quality assurance`_ and
-`modularity`_.
-
-Quality assurance
------------------
-
-The primary aim of `SQLFluff` as a project is in service of that first
-aim of `quality assurance`_. With larger and larger teams maintaining
-large bodies of SQL code, it becomes more and more important that the
-code is not just *valid* but also easily *comprehensible* by other users
-of the same codebase. One way to ensure readability is to enforce a
-`consistent style`_, and the tools used to do this are called `linters`_.
-
-Some famous `linters`_ which are well known in the software community are
-`flake8`_ and `jslint`_ (the former is used to lint the `SQLFluff` project
-itself).
-
-**SQLFluff** aims to fill this space for SQL.
-
-Modularity
-----------
-
-SQL itself doesn't lend itself well to `modularity`_, so to introduce
-some flexibility and reusability it is often `templated`_. Typically
-this is done in the wild in one of the following ways:
-
-1. Using the limited inbuilt templating abilities of a programming
-   language directly. For example in python this would be using the
-   `format string syntax`_:
-
-   .. code-block:: python
-
-      "SELECT {foo} FROM {tbl}".format(foo="bar", tbl="mytable")
-
-   Which would evaluate to:
-
-   .. code-block:: sql
-
-      SELECT bar FROM mytable
-
-2. Using a dedicated templating library such as `jinja2`_. This allows
-   a lot more flexibility and more powerful expressions and macros. See
-   the :ref:`templateconfig` section for more detail on how this works.
-
-   - Often there are tools like `dbt`_ or `apache airflow`_ which allow
-     `templated`_ sql to be used directly, and they will implement a
-     library like `jinja2`_ under the hood themselves.
-
-
-All of these templating tools are great for `modularity`_ but they also
-mean that the SQL files themselves are no longer valid SQL code, because
-they now contain these configured *placeholder* values, intended to
-improve modularity.
-
-SQLFluff supports both of the templating methods outlined above,
-as well as `dbt`_ projects, to allow you to still lint these
-"dynamic" SQL files as part of your CI/CD pipeline (which is great 🙌),
-rather than waiting until you're in production (which is bad 🤦,
-and maybe too late).
-
-During the CI/CD pipeline (or any time that we need to handle `templated`_
-code), SQLFluff needs additional info in order to interpret your templates
-as valid SQL code. You do so by providing dummy parameters in SQLFluff
-configuration files. When substituted into the template, these values should
-evaluate to valid SQL (so SQLFluff can check its style, formatting, and
-correctness), but the values don't need to match actual values used in
-production. This means that you can use *much simpler* dummy values than
-what you would really use. The recommendation is to use *the simplest*
-possible dummy value that still allows your code to evaluate to valid SQL
-so that the configuration values can be a streamlined as possible.
-
-.. _`infrastructure as code`: https://en.wikipedia.org/wiki/Infrastructure_as_code
-.. _`viewpoint section of the docs for the open-source tool dbt`: https://docs.getdbt.com/docs/viewpoint
-.. _`quality assurance`: https://docs.getdbt.com/docs/viewpoint#section-quality-assurance
-.. _`modularity`: https://docs.getdbt.com/docs/viewpoint#section-modularity
-.. _`consistent style`: https://www.smashingmagazine.com/2012/10/why-coding-style-matters/
-.. _`linters`: https://en.wikipedia.org/wiki/Lint_(software)
-.. _`flake8`: http://flake8.pycqa.org/
-.. _`jslint`: https://www.jslint.com/
-.. _`templated`: https://en.wikipedia.org/wiki/Template_processor
-.. _`format string syntax`: https://docs.python.org/3/library/string.html#formatstrings
-.. _`jinja2`: https://jinja.palletsprojects.com/
-.. _`apache airflow`: https://airflow.apache.org
-.. _`dbt`: https://getdbt.com

docs/source/reference/api.rst

@@ -0,0 +1,65 @@
+.. _apiref:
+
+Python API
+==========
+
+SQLFluff exposes a public api for other python applications to use.
+A basic example of this usage is given here, with the documentation
+for each of the methods below.
+
+
+.. literalinclude:: ../../../examples/01_basic_api_usage.py
+   :language: python
+
+
+Simple API commands
+-------------------
+
+
+.. automodule:: sqlfluff
+   :members: lint, fix, parse
+
+
+Advanced API usage
+------------------
+
+The simple API presents only a fraction of the functionality present
+within the core SQLFluff library. For more advanced use cases, users
+can import the :code:`Linter()` and :code:`FluffConfig()` classes from
+:code:`sqlfluff.core`. As of version 0.4.0 this is considered as
+*experimental only* as the internals may change without warning in any
+future release. If you come to rely on the internals of SQLFluff, please
+post an issue on GitHub to share what you're up to. This will help shape
+a more reliable, tidy and well documented public API for use.
+
+Configuring SQLFluff
+~~~~~~~~~~~~~~~~~~~~
+
+You can use :code:`FluffConfig()` class to configure SQLFluff behaviour.
+
+
+.. literalinclude:: ../../../examples/04_config_overrides.py
+   :language: python
+
+
+Instances of :code:`FluffConfig()` can be created manually, or parsed.
+
+
+.. literalinclude:: ../../../examples/05_simple_api_config.py
+   :language: python
+
+
+Supported dialects and rules are available through :code:`list_dialects()`
+and :code:`list_rules()`.
+
+
+.. literalinclude:: ../../../examples/03_getting_rules_and_dialects.py
+   :language: python
+
+
+Advanced API reference
+~~~~~~~~~~~~~~~~~~~~~~
+
+
+.. automodule:: sqlfluff.core
+   :members: Linter, Lexer, Parser

docs/source/reference/dialects.rst

@@ -0,0 +1,48 @@
+.. _dialectref:
+
+Dialects Reference
+==================
+
+SQLFluff is designed to be flexible in supporting a variety of dialects.
+Not all potential dialects are supported so far, but several have been
+implemented by the community. Below are a list of the currently available
+dialects. Each inherits from another, up to the root `ansi` dialect.
+
+For a canonical list of supported dialects, run the
+:program:`sqlfluff dialects` command, which will output a list of the
+current dialects available on your installation of SQLFluff.
+
+.. note::
+
+    For technical users looking to add new dialects or add new features
+    to existing ones, the dependent nature of how dialects have been
+    implemented is to try and reduce the amount of repetition in how
+    different elements are defined. As an example, when we say that
+    the :ref:`redshift_dialect_ref` dialect *inherits* from the
+    :ref:`postgres_dialect_ref` dialect this is not because there
+    is an agreement between those projects which means that features
+    in one must end up in the other, but that the design of the
+    :ref:`redshift_dialect_ref` dialect was heavily *inspired* by the
+    postgres dialect and therefore when defining the dialect within
+    sqlfuff it makes sense to use :ref:`postgres_dialect_ref` as a
+    starting point rather than starting from scratch.
+
+    Consider when adding new features to a dialect:
+
+    - Should I be adding it just to this dialect, or adding it to
+      a *parent* dialect?
+    - If I'm creating a new dialect, which dialect would be best to
+      inherit from?
+    - Will the feature I'm adding break any *downstream* dependencies
+      within dialects which inherit from this one?
+
+.. We define a shortcut to render double backticks here, which can
+   then be referenced by individual dialects when they want to say
+   how backtick quotes behave in that dialect. They would otherwise
+   be interpreted as markup and so not shown as back quotes.
+
+.. |back_quotes| raw:: html
+
+    <code class="code docutils literal notranslate">``</code>
+
+.. include:: ../_partials/dialect_summaries.rst

docs/source/reference/index.rst

@@ -0,0 +1,14 @@
+.. _reference:
+
+Reference
+=========
+
+.. toctree::
+   :maxdepth: 2
+
+   dialects
+   rules
+   cli
+   api
+   internals/index
+   releasenotes