Qual: Add pre-commit to CI (#5730)

This commit is contained in:
MDW
2024-04-01 23:19:41 +02:00
committed by GitHub
parent f400d6d1b2
commit 127e07e8ad
230 changed files with 343 additions and 277 deletions

View File

@@ -20,6 +20,9 @@ indent_style = unset
# Some specific tests with trailing newlines
# If adding any exceptions here, make sure to add them to .pre-commit-config.yaml as well
[test/fixtures/templater/jinja_l_metas/0{01,03,04,05,07,08,11}.sql]
indent_style = unset
insert_final_newline = unset
[test/fixtures/linter/sqlfluffignore/*/*.sql]
indent_style = unset
trim_trailing_whitespace = unset
@@ -32,6 +35,9 @@ insert_final_newline = unset
indent_style = unset
trim_trailing_whitespace = unset
insert_final_newline = unset
[plugins/sqlfluff-templater-dbt/test/fixtures/dbt/dbt_project/models/my_new_project/multiple_trailing_newline.sql]
indent_style = unset
insert_final_newline = unset
[plugins/sqlfluff-templater-dbt/test/fixtures/dbt/templated_output/macro_in_macro.sql]
indent_style = unset
trim_trailing_whitespace = unset

53
.github/workflows/pre-commit.yml vendored Normal file
View File

@@ -0,0 +1,53 @@
---
name: pre-commit
on:
pull_request:
push:
jobs:
pre-commit:
runs-on: ubuntu-latest
env:
RAW_LOG: pre-commit.log
CS_XML: pre-commit.xml
SKIP: no-commit-to-branch
steps:
- run: sudo apt-get update && sudo apt-get install cppcheck
if: false
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
if: false
with:
cache: pip
python-version: 3.12.1
- run: python -m pip install pre-commit
- uses: actions/cache/restore@v4
with:
path: ~/.cache/pre-commit/
key: pre-commit-4|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml')
}}
- name: Run pre-commit hooks
run: |
set -o pipefail
pre-commit gc
pre-commit run --show-diff-on-failure --color=always --all-files | tee ${RAW_LOG}
- name: Convert Raw Log to Checkstyle format (launch action)
uses: mdeweerd/logToCheckStyle@v2024.3.5
if: ${{ failure() }}
with:
in: ${{ env.RAW_LOG }}
out: ${{ env.CS_XML }}
- uses: actions/cache/save@v4
if: ${{ ! cancelled() }}
with:
path: ~/.cache/pre-commit/
key: pre-commit-4|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml')
}}
- name: Provide log as artifact
uses: actions/upload-artifact@v4
if: ${{ ! cancelled() }}
with:
name: precommit-logs
path: |
${{ env.RAW_LOG }}
${{ env.CS_XML }}
retention-days: 2

View File

@@ -2,15 +2,22 @@ repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.5.0
hooks:
- id: no-commit-to-branch
args: [--branch, main]
# If adding any exceptions here, make sure to add them to .editorconfig as well
- id: end-of-file-fixer
exclude: |
(?x)^(
test/fixtures/linter/sqlfluffignore/|
test/fixtures/config/inheritance_b/example.sql|
test/fixtures/config/inheritance_b/nested/example.sql|
plugins/sqlfluff-templater-dbt/test/fixtures/dbt/templated_output/trailing_newlines.sql|
plugins/sqlfluff-templater-dbt/test/fixtures/dbt/dbt_project/models/my_new_project/trailing_newlines.sql
(?x)^
(
test/fixtures/templater/jinja_l_metas/0(0[134578]|11).sql|
test/fixtures/linter/sqlfluffignore/[^/]*/[^/]*.sql|
test/fixtures/config/inheritance_b/(nested/)?example.sql|
(.*)/trailing_newlines.sql|
plugins/sqlfluff-templater-dbt/test/fixtures/dbt/dbt_project/models/my_new_project/multiple_trailing_newline.sql|
plugins/sqlfluff-templater-dbt/test/fixtures/dbt/templated_output/macro_in_macro.sql|
plugins/sqlfluff-templater-dbt/test/fixtures/dbt/templated_output/(dbt_utils_0.8.0/)?last_day.sql|
test/fixtures/linter/indentation_errors.sql|
test/fixtures/templater/jinja_d_roundtrip/test.sql
)$
- id: trailing-whitespace
exclude: |
@@ -19,7 +26,7 @@ repos:
test/fixtures/templater/jinja_d_roundtrip/test.sql|
test/fixtures/config/inheritance_b/example.sql|
test/fixtures/config/inheritance_b/nested/example.sql|
plugins/sqlfluff-templater-dbt/test/fixtures/dbt/templated_output/macro_in_macro.sq|
plugins/sqlfluff-templater-dbt/test/fixtures/dbt/templated_output/macro_in_macro.sql|
plugins/sqlfluff-templater-dbt/test/fixtures/dbt/templated_output/last_day.sql|
plugins/sqlfluff-templater-dbt/test/fixtures/dbt/templated_output/dbt_utils_0.8.0/last_day.sql|
test/fixtures/linter/sqlfluffignore/
@@ -65,3 +72,9 @@ repos:
rev: "v0.3.2"
hooks:
- id: ruff
- repo: https://github.com/codespell-project/codespell
rev: v2.2.6
hooks:
- id: codespell
exclude: (?x)^(test/fixtures/.*|pyproject.toml)$
additional_dependencies: [tomli]

View File

@@ -242,7 +242,7 @@ There are two way to utilize SQLFluff to annotate Github PRs.
:code:`github-annotation-native`) is used. This is a not something that SQLFluff
can control itself and so we currently recommend using the the second option
above and the `action from yuzutech`_.
There is an `open feature request <https://github.com/orgs/community/discussions/68471>`_
for GitHub Actions which you can track to follow this issue.

View File

@@ -1,5 +1,3 @@
{{ config(materialized='ephemeral') }}
select 1 as id

View File

@@ -5,4 +5,3 @@ from cows
order by
birth_date asc,
name desc

View File

@@ -164,7 +164,7 @@ forbidden_modules = [
]
[[tool.importlinter.contracts]]
name = "Helper methods must be internally indepentent"
name = "Helper methods must be internally independent"
type = "independence"
modules = [
"sqlfluff.core.helpers.string",
@@ -253,3 +253,28 @@ markers = [
[tool.doc8]
# Ignore auto-generated docs
ignore-path = "docs/source/partials/"
[tool.codespell]
# The configuration must be kept here to ensure that
# `codespell` can be run as a standalone program from the CLI
# with the appropriate default options.
skip = "*/test/fixtures/*,*/.*,*/pyproject.toml"
check-hidden = true
quiet-level=2
# ignore-regex = '\\[fnrstv]'
builtin = "clear,rare,informal,names"
ignore-words-list = "fo,ws,falsy,coo,inout,deque,crate,trough,ro,mange,identifers,statment"
# ignore-words = "dev/tools/codespell/codespell-ignore.txt"
# exclude-file = "dev/tools/codespell/codespell-lines-ignore.txt"
uri-ignore-words-list="crate"
# For future reference: it is not currently possible to specify
# the standard dictionary and the custom dictionary in the configuration
# file
# D = "-"
# dictionary = "dev/tools/codespell/codespell-dict.txt"

View File

@@ -1 +1 @@
SELECT 1 FROM tbl
SELECT 1 FROM tbl

View File

@@ -7,4 +7,4 @@ foo=barbar
[sqlfluff:templater:placeholder]
param_style = flyway_var
flyway:database = test_db
flyway:database = test_db

View File

@@ -1 +1 @@
ALTER TABLE old_table_name RENAME TO new_table_name;
ALTER TABLE old_table_name RENAME TO new_table_name;

View File

@@ -2,4 +2,4 @@ select a
/*
)
*/
from b
from b

View File

@@ -1 +1 @@
create database my_database
create database my_database

View File

@@ -1 +1 @@
create database if not exists my_database
create database if not exists my_database

View File

@@ -1 +1 @@
CREATE ROLE foo_role
CREATE ROLE foo_role

View File

@@ -1 +1 @@
create schema my_schema
create schema my_schema

View File

@@ -1 +1 @@
create schema if not exists my_schema
create schema if not exists my_schema

View File

@@ -2,4 +2,4 @@ CREATE OR REPLACE TABLE t2 AS
SELECT
a,
b
FROM t1
FROM t1

View File

@@ -1,3 +1,3 @@
CREATE TABLE a (
ts TIMESTAMP DEFAULT GETDATE()
)
)

View File

@@ -7,4 +7,3 @@ CREATE TRIGGER foo AFTER UPDATE OF bar, baz ON bar EXECUTE PROCEDURE proc(args);
CREATE TRIGGER foo INSTEAD OF DELETE ON bar FROM baz DEFERRABLE INITIALLY IMMEDIATE FOR EACH ROW EXECUTE PROCEDURE proc(args);
CREATE TRIGGER foo BEFORE INSERT ON bar WHEN (a=b) EXECUTE PROCEDURE proc(args);

View File

@@ -1,4 +1,4 @@
-- Snowflake Double-Dot Notation
-- https://docs.snowflake.com/en/sql-reference/name-resolution.html#resolution-when-schema-omitted-double-dot-notation
SELECT *
FROM my_database..my_table
FROM my_database..my_table

View File

@@ -1,3 +1,3 @@
SELECT *
FROM x
WHERE z LIKE '^_f' ESCAPE '^'
WHERE z LIKE '^_f' ESCAPE '^'

View File

@@ -1,2 +1 @@
select a from tbl1 /*comment here*/ ; /*and here*/ select b from tbl2; -- trailling ending comment

View File

@@ -11,4 +11,4 @@ with tmp as (
select * from blah
)
select a, b from tmp;
# And that's the end
# And that's the end

View File

@@ -1,4 +1,4 @@
select
*
from boo;
WITH blah AS (select x,y,4.567 FROM foo) select z, y, x from blah;
WITH blah AS (select x,y,4.567 FROM foo) select z, y, x from blah;

View File

@@ -1 +1 @@
select a,b, c from sch."blah"
select a,b, c from sch."blah"

View File

@@ -2,4 +2,4 @@
SELECT
a AND NOT i.b
FROM i
FROM i

View File

@@ -1 +1 @@
select * from foo JOIN bar ON (foo.a = bar.a)
select * from foo JOIN bar ON (foo.a = bar.a)

View File

@@ -6,4 +6,4 @@ SELECT
END
) AS tech_support
FROM
zendesk
zendesk

View File

@@ -17,4 +17,4 @@ b as (
)
select *
from b
from b

View File

@@ -11,4 +11,4 @@ SELECT
ELSE
0
END
churn
churn

View File

@@ -5,4 +5,4 @@ SELECT
FROM
dat
WHERE
c >= (SELECT 1)
c >= (SELECT 1)

View File

@@ -18,4 +18,4 @@ JOIN
ON
(business_type = low_business_type)
AND (business_type = low_business_type
OR size_label = low_size_label);
OR size_label = low_size_label);

View File

@@ -8,4 +8,4 @@ FROM
FULL JOIN
business_types AS exists_right
ON
exists_left.business_type = exists_right.business_type
exists_left.business_type = exists_right.business_type

View File

@@ -4,4 +4,4 @@ FROM Produce
WINDOW
a AS (PARTITION BY category),
b AS (ORDER BY purchases),
c AS (ROWS BETWEEN 2 PRECEDING AND 2 FOLLOWING)
c AS (ROWS BETWEEN 2 PRECEDING AND 2 FOLLOWING)

View File

@@ -1,4 +1,4 @@
-- query with no brackets
-- query with no brackets
select
orders.order_id AS order_id,
customers.email AS email
@@ -93,4 +93,4 @@ group by
orders.order_id,
customers.email
order by
orders.order_id;
orders.order_id;

View File

@@ -1,2 +1,2 @@
SELECT (DATE '2001-02-16', DATE '2001-12-21')
OVERLAPS (DATE '2001-10-30', DATE '2002-10-30');
OVERLAPS (DATE '2001-10-30', DATE '2002-10-30');

View File

@@ -5,4 +5,4 @@ SELECT
SAFE_CAST(NULL AS STRING) AS age_label,
EXTRACT(day FROM end_time) AS day
FROM
benchmark_with_performance
benchmark_with_performance

View File

@@ -11,4 +11,4 @@ ON
LEFT JOIN
"other_db"."other_data" AS od
ON
od.fid = data_max.fid
od.fid = data_max.fid

View File

@@ -10,4 +10,4 @@ WITH result AS (
FROM sales_us AS s2
)
SELECT * FROM result
SELECT * FROM result

View File

@@ -1,4 +1,4 @@
SELECT column_name
FROM table1
RIGHT JOIN table2
ON table1.column_name = table2.column_name
ON table1.column_name = table2.column_name

View File

@@ -1 +1 @@
select 1
select 1

View File

@@ -1 +1 @@
select * from blah
select * from blah

View File

@@ -1 +1 @@
select * from foo, bar
select * from foo, bar

View File

@@ -1 +1 @@
select 12 -- ends with comment
select 12 -- ends with comment

View File

@@ -3,4 +3,4 @@ SELECT TRUE AND NOT FALSE;
SELECT TRUE;
SELECT TRUE AND FALSE;
SELECT NOT TRUE;
SELECT NOT TRUE AND FALSE;
SELECT NOT TRUE AND FALSE;

View File

@@ -1,2 +1,2 @@
WITH cte as (select a from tbla)
select a from cte
select a from cte

View File

@@ -1 +1 @@
WITH blah AS (select x,y,z FROM foo) select z, y, x from blah;
WITH blah AS (select x,y,z FROM foo) select z, y, x from blah;

View File

@@ -1,3 +1,3 @@
SELECT *
FROM counter
LIMIT 10 OFFSET 10
LIMIT 10 OFFSET 10

View File

@@ -1,3 +1,3 @@
SELECT *
FROM counter
LIMIT 10, 10
LIMIT 10, 10

View File

@@ -1,3 +1,3 @@
SELECT *
FROM counter
LIMIT 10
LIMIT 10

View File

@@ -1 +1 @@
set schema my_schema
set schema my_schema

View File

@@ -1,4 +1,4 @@
select
'1' :: INT as id1,
'2'::int as id2
from table_a
from table_a

View File

@@ -1,3 +1,3 @@
BEGIN TRANSACTION;
UPDATE tbl SET foo = 1 WHERE bar = 2;
END TRANSACTION;
END TRANSACTION;

View File

@@ -1,3 +1,3 @@
create view my_schema.my_view as
select * from schema.table
with no schema binding;
with no schema binding;

View File

@@ -18,4 +18,3 @@ SELECT
approvalStatus varchar
)
) as test;

View File

@@ -5,4 +5,4 @@ FROM
`database.schema.benchmark_user_map`
WHERE
list_id IS NULL
OR user_id IS NULL
OR user_id IS NULL

View File

@@ -2,5 +2,3 @@ SELECT
gcpproject.functions.timestamp_parsing(log_tbl.orderdate) AS orderdate
FROM
`gcp-project.data.year_2021` AS log_tbl

View File

@@ -7,4 +7,4 @@ SELECT SUM(CASE WHEN value != previous_value THEN 1.0 ELSE 0.0 END)
CASE WHEN ix != 0 THEN LAG(value) OVER (ORDER BY ix ASC) ELSE value END AS previous_value
FROM UNNEST(sequence_validation_and_business_rules.sequence_validation_and_business_rules) AS value
WITH OFFSET AS ix
)
)

View File

@@ -14,4 +14,3 @@ SELECT
planet."Amount of sand"
FROM (SELECT {name: 'Tatooine', 'Amount of sand': 'High'} as planet)
;

View File

@@ -2,4 +2,4 @@ ALTER DATABASE foo SET DBPROPERTIES ("prop1"="val1", "prop2"="val2");
ALTER DATABASE bar SET LOCATION "hdfs://alternate_path";
ALTER DATABASE foo2 SET OWNER USER "user";
ALTER DATABASE foo2 SET OWNER USER "user";

View File

@@ -1,3 +1,3 @@
ALTER DATABASE foo SET OWNER ROLE "role";
ALTER DATABASE bar SET MANAGEDLOCATION "hdfs://alternate_path";
ALTER DATABASE bar SET MANAGEDLOCATION "hdfs://alternate_path";

View File

@@ -1 +1 @@
CREATE DATABASE foo LOCATION 'hdfs://path' WITH DBPROPERTIES ("a"="1", "b"="2");
CREATE DATABASE foo LOCATION 'hdfs://path' WITH DBPROPERTIES ("a"="1", "b"="2");

View File

@@ -1 +1 @@
CREATE SCHEMA IF NOT EXISTS foo COMMENT 'test schema' MANAGEDLOCATION 'hdfs://path';
CREATE SCHEMA IF NOT EXISTS foo COMMENT 'test schema' MANAGEDLOCATION 'hdfs://path';

View File

@@ -3,4 +3,4 @@ CREATE TABLE new_foo
STORED AS RCFile
AS
SELECT (col1 % 1024) col, concat(col1, col2) col12
FROM foo;
FROM foo;

View File

@@ -2,4 +2,4 @@ CREATE TABLE IF NOT EXISTS foo (
col1 string,
col2 float
)
CLUSTERED BY (col2) SORTED BY (col1 DESC) INTO 5 BUCKETS;
CLUSTERED BY (col2) SORTED BY (col1 DESC) INTO 5 BUCKETS;

View File

@@ -7,4 +7,4 @@ CREATE TABLE db.foo (
col6 MAP<varchar, date>,
col7 STRUCT< field1: boolean, field2: ARRAY<double precision>, field3: UNIONTYPE<string, decimal(10, 2)>>,
col8 UNIONTYPE<string, ARRAY<char>>
);
);

View File

@@ -3,4 +3,4 @@ CREATE EXTERNAL TABLE IF NOT EXISTS foo (
col2 string
)
PARTITIONED BY (col3 string, col4 date)
LOCATION 'hdfs://path';
LOCATION 'hdfs://path';

View File

@@ -1,3 +1,3 @@
CREATE TABLE db.new_foo
LIKE foo
TBLPROPERTIES ("property_name"="property_value");
TBLPROPERTIES ("property_name"="property_value");

View File

@@ -8,4 +8,4 @@ WITH SERDEPROPERTIES (
"input.regex" = "([^]*) ([^]*) ([^]*) (-|\\[^\\]*\\]) ([^ \"]*|\"[^\"]*\") (-|[0-9]*) (-|[0-9]*)(?: ([^ \"]*|\".*\") ([^ \"]*|\".*\"))?"
)
STORED AS TEXTFILE
LOCATION 'hdfs://path';
LOCATION 'hdfs://path';

View File

@@ -1,2 +1,2 @@
CREATE TABLE foo (col1 STRING, col2 int, col3 STRING)
SKEWED BY (col1, col2) ON (('s1',1), ('s3',3), ('s13',13), ('s78',78)) STORED AS DIRECTORIES;
SKEWED BY (col1, col2) ON (('s1',1), ('s3',3), ('s13',13), ('s78',78)) STORED AS DIRECTORIES;

View File

@@ -2,4 +2,4 @@ CREATE TEMPORARY TABLE foo (
col1 string,
col2 float
)
TBLPROPERTIES ("property1"="value1", "property2"="value2");
TBLPROPERTIES ("property1"="value1", "property2"="value2");

View File

@@ -1,3 +1,3 @@
DROP TABLE foo;
DROP TABLE IF exists bar PURGE;
DROP TABLE IF exists bar PURGE;

View File

@@ -3,4 +3,4 @@ set -v;
set foo = 2;
set foo = 'bar';
set hivevar:cat="Chloe";
set mapreduce.reduce.memory.mb=12000;
set mapreduce.reduce.memory.mb=12000;

View File

@@ -1,3 +1,3 @@
TRUNCATE TABLE foo;
TRUNCATE bar PARTITION (col='val');
TRUNCATE bar PARTITION (col='val');

View File

@@ -1 +1 @@
USE db;
USE db;

View File

@@ -17,4 +17,4 @@ CREATE SINK frank_quotes_cluster
FROM frank_quotes
INTO KAFKA CONNECTION kafka_connection (TOPIC 'frank-quotes-sink')
FORMAT JSON
ENVELOPE DEBEZIUM;
ENVELOPE DEBEZIUM;

View File

@@ -4,4 +4,4 @@ GRANT USAGE, CREATE ON DATABASE materialize TO joe;
GRANT ALL ON CLUSTER dev TO joe;
GRANT CREATEDB ON SYSTEM TO joe;
GRANT CREATEDB ON SYSTEM TO joe;

View File

@@ -1,3 +1,3 @@
blocks:BEGIN
select 1;
END blocks~
END blocks~

View File

@@ -1 +1 @@
CALL somefunction('a');
CALL somefunction('a');

View File

@@ -1 +1 @@
CALL somefunction(test('something'));
CALL somefunction(test('something'));

View File

@@ -1 +1 @@
CALL somefunction('test', @test1, test2, test3('test'), "test4");
CALL somefunction('test', @test1, test2, test3('test'), "test4");

View File

@@ -1 +1 @@
CALL somefunction();
CALL somefunction();

View File

@@ -1 +1 @@
CALL `somefunction`('a');
CALL `somefunction`('a');

View File

@@ -1 +1 @@
CLOSE curcursor;
CLOSE curcursor;

View File

@@ -1 +1 @@
CLOSE `curcursor`;
CLOSE `curcursor`;

View File

@@ -4,4 +4,4 @@ CREATE TABLE IF NOT EXISTS db_name.table_name
updated_at2 timestamp not null default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP,
updated_at3 timestamp default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP not null,
updated_at4 timestamp
);
);

View File

@@ -1 +1 @@
DEALLOCATE PREPARE dynam;
DEALLOCATE PREPARE dynam;

View File

@@ -1 +1 @@
DECLARE random_condition_name CONDITION FOR 1051;
DECLARE random_condition_name CONDITION FOR 1051;

View File

@@ -1,4 +1,4 @@
DECLARE continue handler for sqlexception
begin
select 1;
end;
end;

View File

@@ -1 +1 @@
DECLARE test CURSOR FOR SELECT 1;
DECLARE test CURSOR FOR SELECT 1;

View File

@@ -1 +1 @@
DECLARE abc int default 1;
DECLARE abc int default 1;

View File

@@ -1 +1 @@
DECLARE abc longtext default 'test';
DECLARE abc longtext default 'test';

View File

@@ -1,4 +1,4 @@
DECLARE exit handler for sqlexception
begin
select 1;
end;
end;

View File

@@ -1 +1 @@
DECLARE abc int;
DECLARE abc int;

View File

@@ -1,4 +1,4 @@
DECLARE undo handler for sqlexception
begin
select 1;
end;
end;

View File

@@ -8,4 +8,4 @@ SQL SECURITY DEFINER
BEGIN
SELECT 1 + 2;
END~
DELIMITER ;
DELIMITER ;

View File

@@ -7,4 +7,4 @@ SQL SECURITY DEFINER
BEGIN
SELECT 1 + 2;
END~
DELIMITER ;
DELIMITER ;

View File

@@ -1,3 +1,3 @@
DELIMITER ~
SELECT 1~
DELIMITER ;
DELIMITER ;

View File

@@ -1 +1 @@
DROP FUNCTION IF EXISTS `testfunction`;
DROP FUNCTION IF EXISTS `testfunction`;

View File

@@ -1 +1 @@
DROP PREPARE dynam;
DROP PREPARE dynam;

Some files were not shown because too many files have changed in this diff Show More