Compare commits

...

6 Commits

Author SHA1 Message Date
Greg Finley
5793de9a72 Merge branch 'main' into issue-4260 2023-05-01 14:42:07 -07:00
WittierDinosaur
b5635e78a2 YML gen 2023-04-27 14:14:41 +00:00
WittierDinosaur
2d21ee46d6 lint 2023-04-27 14:06:59 +00:00
WittierDinosaur
20c8aa4a0b Remove Postgres hash lex from Redshift 2023-04-27 14:04:48 +00:00
WittierDinosaur
2e27363c6a Linting 2023-04-27 13:17:02 +00:00
WittierDinosaur
477711a2a6 Postgres: Add Support for PostGIS operators 2023-04-27 13:10:38 +00:00
8 changed files with 2259 additions and 25 deletions

View File

@@ -344,6 +344,20 @@ class Dialect:
# Overwrite with the buffer once we're done
self.lexer_matchers = buff
def delete_lexer_matchers(self, lexer_patch):
"""Given a list of lexer names, delete them."""
if not self.lexer_matchers: # pragma: no cover
raise ValueError("Lexer struct must be defined before it can be patched!")
buff = []
patch_set = set(lexer_patch)
for elem in self.lexer_matchers:
if elem.name not in patch_set:
buff.append(elem)
self.lexer_matchers = buff
def insert_lexer_matchers(self, lexer_patch, before):
"""Insert new records into an existing lexer struct.

View File

@@ -45,6 +45,7 @@ postgres_dialect.insert_lexer_matchers(
# JSON Operators: https://www.postgresql.org/docs/9.5/functions-json.html
[
StringLexer("right_arrow", "=>", CodeSegment),
StringLexer("hash", "#", CodeSegment),
],
before="equals",
)
@@ -104,12 +105,6 @@ postgres_dialect.insert_lexer_matchers(
CodeSegment,
segment_kwargs={"type": "unicode_double_quote"},
),
RegexLexer(
"json_operator",
r"->>|#>>|->|#>|@>|<@|\?\||\?|\?&|#-",
SymbolSegment,
segment_kwargs={"type": "json_operator"},
),
StringLexer("at", "@", CodeSegment),
# https://www.postgresql.org/docs/current/sql-syntax-lexical.html
RegexLexer(
@@ -230,9 +225,6 @@ postgres_dialect.sets("date_part_function_name").clear()
postgres_dialect.sets("value_table_functions").update(["UNNEST", "GENERATE_SERIES"])
postgres_dialect.add(
JsonOperatorSegment=TypedParser(
"json_operator", SymbolSegment, type="binary_operator"
),
SimpleGeometryGrammar=AnyNumberOf(Ref("NumericLiteralSegment")),
# N.B. this MultilineConcatenateDelimiterGrammar is only created
# to parse multiline-concatenated string literals
@@ -282,9 +274,31 @@ postgres_dialect.add(
Sequence(Ref("TableReferenceSegment"), Ref("StarSegment")),
),
RightArrowSegment=StringParser("=>", SymbolSegment, type="right_arrow"),
AtSegment=StringParser("@", SymbolSegment, type="at"),
HashSegment=StringParser("#", SymbolSegment, type="hash"),
OnKeywordAsIdentifierSegment=StringParser(
"ON", ansi.IdentifierSegment, type="naked_identifier"
),
PostgisOperatorGrammar=OneOf(
Ref("IntersectsSegment"),
Ref("NDIntersectsSegment"),
Ref("OverlapsLeftSegment"),
Ref("OverlapsBelowSegment"),
Ref("OverlapsRightSegment"),
Ref("OverlapsAboveSegment"),
Ref("StrictlyLeftSegment"),
Ref("StrictlyBelowSegment"),
Ref("StrictlyRightSegment"),
Ref("StrictlyAboveSegment"),
Ref("ContainedSegment"),
Ref("ContainsSegment"),
Ref("SameSegment"),
Ref("DistanceSegment"),
Ref("ClosestSegment"),
Ref("BoxDistanceSegment"),
Ref("NDDistanceSegment"),
Ref("NDBoxDistanceSegment"),
),
)
postgres_dialect.replace(
@@ -304,6 +318,7 @@ postgres_dialect.replace(
Ref("NotExtendRightSegment"),
Ref("NotExtendLeftSegment"),
Ref("AdjacentSegment"),
Ref("PostgisOperatorGrammar"),
),
NakedIdentifierSegment=SegmentGenerator(
# Generate the anti template from the set of reserved keywords
@@ -523,6 +538,213 @@ postgres_dialect.replace(
)
class JsonOperatorSegment(BaseSegment):
"""A JSON operator."""
type = "json_operator"
match_grammar = OneOf(
Sequence(
OneOf(
Ref("MinusSegment"),
Ref("HashSegment"),
),
Ref("GreaterThanSegment"),
Ref("GreaterThanSegment", optional=True),
allow_gaps=False,
),
Sequence(Ref("AtSegment"), Ref("GreaterThanSegment"), allow_gaps=False),
Sequence(
Ref("LessThanSegment"),
Ref("AtSegment"),
allow_gaps=False,
),
Sequence(
Ref("ParameterSegment"),
OneOf(
Ref("PipeSegment"),
Ref("AmpersandSegment"),
),
allow_gaps=False,
),
Sequence(
Ref("HashSegment"),
Ref("MinusSegment"),
),
)
class IntersectsSegment(ansi.CompositeComparisonOperatorSegment):
"""Postgis Intersects."""
match_grammar = Sequence(
Ref("AmpersandSegment"), Ref("AmpersandSegment"), allow_gaps=False
)
class NDIntersectsSegment(ansi.CompositeComparisonOperatorSegment):
"""Postgis N-D Intersects."""
match_grammar = Sequence(
Ref("AmpersandSegment"),
Ref("AmpersandSegment"),
Ref("AmpersandSegment"),
allow_gaps=False,
)
class OverlapsLeftSegment(ansi.CompositeComparisonOperatorSegment):
"""Postgis Overlaps Left."""
match_grammar = Sequence(
Ref("AmpersandSegment"), Ref("LessThanSegment"), allow_gaps=False
)
class OverlapsBelowSegment(ansi.CompositeComparisonOperatorSegment):
"""Postgis Overlaps Below."""
match_grammar = Sequence(
Ref("AmpersandSegment"),
Ref("LessThanSegment"),
Ref("PipeSegment"),
allow_gaps=False,
)
class OverlapsRightSegment(ansi.CompositeComparisonOperatorSegment):
"""Postgis Overlaps Right."""
match_grammar = Sequence(
Ref("AmpersandSegment"), Ref("GreaterThanSegment"), allow_gaps=False
)
class OverlapsAboveSegment(ansi.CompositeComparisonOperatorSegment):
"""Postgis Overlaps Above."""
match_grammar = Sequence(
Ref("PipeSegment"),
Ref("AmpersandSegment"),
Ref("GreaterThanSegment"),
allow_gaps=False,
)
class StrictlyLeftSegment(ansi.CompositeComparisonOperatorSegment):
"""Postgis Strictly Left."""
match_grammar = Sequence(
Ref("LessThanSegment"), Ref("LessThanSegment"), allow_gaps=False
)
class StrictlyBelowSegment(ansi.CompositeComparisonOperatorSegment):
"""Postgis Strictly Below."""
match_grammar = Sequence(
Ref("LessThanSegment"),
Ref("LessThanSegment"),
Ref("PipeSegment"),
allow_gaps=False,
)
class StrictlyRightSegment(ansi.CompositeComparisonOperatorSegment):
"""Postgis Strictly Right."""
match_grammar = Sequence(
Ref("GreaterThanSegment"), Ref("GreaterThanSegment"), allow_gaps=False
)
class StrictlyAboveSegment(ansi.CompositeComparisonOperatorSegment):
"""Postgis Strcitly Above."""
match_grammar = Sequence(
Ref("PipeSegment"),
Ref("GreaterThanSegment"),
Ref("GreaterThanSegment"),
allow_gaps=False,
)
class ContainedSegment(ansi.CompositeComparisonOperatorSegment):
"""Postgis Contained."""
match_grammar = Ref("AtSegment")
class ContainsSegment(ansi.CompositeComparisonOperatorSegment):
"""Postgis Contains."""
match_grammar = Ref("TildeSegment")
class SameSegment(ansi.CompositeComparisonOperatorSegment):
"""Postgis Same."""
match_grammar = Sequence(
Ref("TildeSegment"), Ref("EqualsSegment"), allow_gaps=False
)
class DistanceSegment(ansi.CompositeComparisonOperatorSegment):
"""Postgis Distance."""
match_grammar = Sequence(
Ref("LessThanSegment"),
Ref("MinusSegment"),
Ref("GreaterThanSegment"),
allow_gaps=False,
)
class ClosestSegment(ansi.CompositeComparisonOperatorSegment):
"""Postgis Distance."""
match_grammar = Sequence(
Ref("PipeSegment"), Ref("EqualsSegment"), Ref("PipeSegment"), allow_gaps=False
)
class BoxDistanceSegment(ansi.CompositeComparisonOperatorSegment):
"""Postgis Distance."""
match_grammar = Sequence(
Ref("LessThanSegment"),
Ref("HashSegment"),
Ref("GreaterThanSegment"),
allow_gaps=False,
)
class NDDistanceSegment(ansi.CompositeComparisonOperatorSegment):
"""Postgis Distance."""
match_grammar = Sequence(
Ref("LessThanSegment"),
Ref("LessThanSegment"),
Ref("MinusSegment"),
Ref("GreaterThanSegment"),
Ref("GreaterThanSegment"),
allow_gaps=False,
)
class NDBoxDistanceSegment(ansi.CompositeComparisonOperatorSegment):
"""Postgis Distance."""
match_grammar = Sequence(
Ref("LessThanSegment"),
Ref("LessThanSegment"),
Ref("HashSegment"),
Ref("GreaterThanSegment"),
Ref("GreaterThanSegment"),
allow_gaps=False,
)
# Inherit from the ANSI ObjectReferenceSegment this way so we can inherit
# other segment types from it.
class ObjectReferenceSegment(ansi.ObjectReferenceSegment):

View File

@@ -202,6 +202,9 @@ redshift_dialect.patch_lexer_matchers(
)
redshift_dialect.delete_lexer_matchers(["hash"])
# Inherit from the Postgres ObjectReferenceSegment this way so we can inherit
# other segment types from it.
class ObjectReferenceSegment(postgres.ObjectReferenceSegment):

View File

@@ -3,7 +3,7 @@
# computed by SQLFluff when running the tests. Please run
# `python test/generate_parse_fixture_yml.py` to generate them after adding or
# altering SQL files.
_hash: e4637e97ea11a573f43a5e1d0a19f101f3e569164fa4f0886387efd47fb8053a
_hash: 58bf5fbb3efa396ef277c46668f15f87a3b5ef752484990702536168a9817236
file:
- statement:
create_source_kafka_statement:
@@ -51,7 +51,12 @@ file:
expression:
column_reference:
naked_identifier: data
binary_operator: ->>
json_operator:
- binary_operator: '-'
- comparison_operator:
raw_comparison_operator: '>'
- comparison_operator:
raw_comparison_operator: '>'
quoted_literal: "'field1'"
alias_expression:
keyword: AS
@@ -61,7 +66,12 @@ file:
expression:
column_reference:
naked_identifier: data
binary_operator: ->>
json_operator:
- binary_operator: '-'
- comparison_operator:
raw_comparison_operator: '>'
- comparison_operator:
raw_comparison_operator: '>'
quoted_literal: "'field2'"
alias_expression:
keyword: AS
@@ -71,7 +81,12 @@ file:
expression:
column_reference:
naked_identifier: data
binary_operator: ->>
json_operator:
- binary_operator: '-'
- comparison_operator:
raw_comparison_operator: '>'
- comparison_operator:
raw_comparison_operator: '>'
quoted_literal: "'field3'"
alias_expression:
keyword: AS

View File

@@ -3,7 +3,7 @@
# computed by SQLFluff when running the tests. Please run
# `python test/generate_parse_fixture_yml.py` to generate them after adding or
# altering SQL files.
_hash: 48a081d4a74ecbf99ac89dadf074297fb904014b8f2b684a8e5b927ccc7cc7a0
_hash: 3f5ee3e7d0d2a932c1f54eac4faef3c79a4d3a02df6a0c37e9e9bd0cc9267b0e
file:
- statement:
select_statement:
@@ -18,7 +18,12 @@ file:
- expression:
column_reference:
naked_identifier: doc
binary_operator: '#>>'
json_operator:
- hash: '#'
- comparison_operator:
raw_comparison_operator: '>'
- comparison_operator:
raw_comparison_operator: '>'
quoted_literal: "'{fields}'"
- comma: ','
- expression:
@@ -39,7 +44,12 @@ file:
expression:
- column_reference:
naked_identifier: doc
- binary_operator: ->>
- json_operator:
- binary_operator: '-'
- comparison_operator:
raw_comparison_operator: '>'
- comparison_operator:
raw_comparison_operator: '>'
- quoted_literal: "'some_field'"
- comparison_operator:
raw_comparison_operator: '='
@@ -56,7 +66,10 @@ file:
casting_operator: '::'
data_type:
keyword: json
binary_operator: ->
json_operator:
binary_operator: '-'
comparison_operator:
raw_comparison_operator: '>'
numeric_literal: '2'
- statement_terminator: ;
- statement:
@@ -70,7 +83,10 @@ file:
casting_operator: '::'
data_type:
keyword: json
binary_operator: ->
json_operator:
binary_operator: '-'
comparison_operator:
raw_comparison_operator: '>'
quoted_literal: "'a'"
- statement_terminator: ;
- statement:
@@ -84,7 +100,12 @@ file:
casting_operator: '::'
data_type:
keyword: json
binary_operator: ->>
json_operator:
- binary_operator: '-'
- comparison_operator:
raw_comparison_operator: '>'
- comparison_operator:
raw_comparison_operator: '>'
numeric_literal: '2'
- statement_terminator: ;
- statement:
@@ -98,7 +119,12 @@ file:
casting_operator: '::'
data_type:
keyword: json
binary_operator: ->>
json_operator:
- binary_operator: '-'
- comparison_operator:
raw_comparison_operator: '>'
- comparison_operator:
raw_comparison_operator: '>'
quoted_literal: "'b'"
- statement_terminator: ;
- statement:
@@ -112,7 +138,10 @@ file:
casting_operator: '::'
data_type:
keyword: json
binary_operator: '#>'
json_operator:
hash: '#'
comparison_operator:
raw_comparison_operator: '>'
quoted_literal: "'{a,b}'"
- statement_terminator: ;
- statement:
@@ -126,6 +155,11 @@ file:
casting_operator: '::'
data_type:
keyword: json
binary_operator: '#>>'
json_operator:
- hash: '#'
- comparison_operator:
raw_comparison_operator: '>'
- comparison_operator:
raw_comparison_operator: '>'
quoted_literal: "'{a,2}'"
- statement_terminator: ;

View File

@@ -0,0 +1,149 @@
SELECT tbl1.column1, tbl2.column1, tbl1.column2 && tbl2.column2 AS overlap
FROM ( VALUES
(1, 'LINESTRING(0 0, 3 3)'::geometry),
(2, 'LINESTRING(0 1, 0 5)'::geometry)) AS tbl1,
( VALUES
(3, 'LINESTRING(1 2, 4 6)'::geometry)) AS tbl2;
SELECT tbl1.column1, tbl2.column1, tbl1.column2 &&& tbl2.column2 AS overlaps_3d,
tbl1.column2 && tbl2.column2 AS overlaps_2d
FROM ( VALUES
(1, 'LINESTRING Z(0 0 1, 3 3 2)'::geometry),
(2, 'LINESTRING Z(1 2 0, 0 5 -1)'::geometry)) AS tbl1,
( VALUES
(3, 'LINESTRING Z(1 2 1, 4 6 1)'::geometry)) AS tbl2;
SELECT tbl1.column1, tbl2.column1, tbl1.column2 &< tbl2.column2 AS overleft
FROM
( VALUES
(1, 'LINESTRING(1 2, 4 6)'::geometry)) AS tbl1,
( VALUES
(2, 'LINESTRING(0 0, 3 3)'::geometry),
(3, 'LINESTRING(0 1, 0 5)'::geometry),
(4, 'LINESTRING(6 0, 6 1)'::geometry)) AS tbl2;
SELECT tbl1.column1, tbl2.column1, tbl1.column2 &<| tbl2.column2 AS overbelow
FROM
( VALUES
(1, 'LINESTRING(6 0, 6 4)'::geometry)) AS tbl1,
( VALUES
(2, 'LINESTRING(0 0, 3 3)'::geometry),
(3, 'LINESTRING(0 1, 0 5)'::geometry),
(4, 'LINESTRING(1 2, 4 6)'::geometry)) AS tbl2;
SELECT tbl1.column1, tbl2.column1, tbl1.column2 &> tbl2.column2 AS overright
FROM
( VALUES
(1, 'LINESTRING(1 2, 4 6)'::geometry)) AS tbl1,
( VALUES
(2, 'LINESTRING(0 0, 3 3)'::geometry),
(3, 'LINESTRING(0 1, 0 5)'::geometry),
(4, 'LINESTRING(6 0, 6 1)'::geometry)) AS tbl2;
SELECT tbl1.column1, tbl2.column1, tbl1.column2 << tbl2.column2 AS strict_left
FROM
( VALUES
(1, 'LINESTRING (1 2, 1 5)'::geometry)) AS tbl1,
( VALUES
(2, 'LINESTRING (0 0, 4 3)'::geometry),
(3, 'LINESTRING (6 0, 6 5)'::geometry),
(4, 'LINESTRING (2 2, 5 6)'::geometry)) AS tbl2;
SELECT tbl1.column1, tbl2.column1, tbl1.column2 <<| tbl2.column2 AS below
FROM
( VALUES
(1, 'LINESTRING (0 0, 4 3)'::geometry)) AS tbl1,
( VALUES
(2, 'LINESTRING (1 4, 1 7)'::geometry),
(3, 'LINESTRING (6 1, 6 5)'::geometry),
(4, 'LINESTRING (2 3, 5 6)'::geometry)) AS tbl2;
SELECT 'LINESTRING(0 0, 0 1, 1 0)'::geometry = 'LINESTRING(1 1, 0 0)'::geometry;
SELECT tbl1.column1, tbl2.column1, tbl1.column2 >> tbl2.column2 AS strict_right
FROM
( VALUES
(1, 'LINESTRING (2 3, 5 6)'::geometry)) AS tbl1,
( VALUES
(2, 'LINESTRING (1 4, 1 7)'::geometry),
(3, 'LINESTRING (6 1, 6 5)'::geometry),
(4, 'LINESTRING (0 0, 4 3)'::geometry)) AS tbl2;
SELECT tbl1.column1, tbl2.column1, tbl1.column2 @ tbl2.column2 AS contained
FROM
( VALUES
(1, 'LINESTRING (1 1, 3 3)'::geometry)) AS tbl1,
( VALUES
(2, 'LINESTRING (0 0, 4 4)'::geometry),
(3, 'LINESTRING (2 2, 4 4)'::geometry),
(4, 'LINESTRING (1 1, 3 3)'::geometry)) AS tbl2;
SELECT tbl1.column1, tbl2.column1, tbl1.column2 |&> tbl2.column2 AS overabove
FROM
( VALUES
(1, 'LINESTRING(6 0, 6 4)'::geometry)) AS tbl1,
( VALUES
(2, 'LINESTRING(0 0, 3 3)'::geometry),
(3, 'LINESTRING(0 1, 0 5)'::geometry),
(4, 'LINESTRING(1 2, 4 6)'::geometry)) AS tbl2;
SELECT tbl1.column1, tbl2.column1, tbl1.column2 |>> tbl2.column2 AS above
FROM
( VALUES
(1, 'LINESTRING (1 4, 1 7)'::geometry)) AS tbl1,
( VALUES
(2, 'LINESTRING (0 0, 4 2)'::geometry),
(3, 'LINESTRING (6 1, 6 5)'::geometry),
(4, 'LINESTRING (2 3, 5 6)'::geometry)) AS tbl2;
SELECT tbl1.column1, tbl2.column1, tbl1.column2 ~ tbl2.column2 AS contains
FROM
( VALUES
(1, 'LINESTRING (0 0, 3 3)'::geometry)) AS tbl1,
( VALUES
(2, 'LINESTRING (0 0, 4 4)'::geometry),
(3, 'LINESTRING (1 1, 2 2)'::geometry),
(4, 'LINESTRING (0 0, 3 3)'::geometry)) AS tbl2;
select 'LINESTRING(0 0, 1 1)'::geometry ~= 'LINESTRING(0 1, 1 0)'::geometry as equality;
SELECT st_distance(geom, 'SRID=3005;POINT(1011102 450541)'::geometry) as d,edabbr, vaabbr
FROM va2005
ORDER BY geom <-> 'SRID=3005;POINT(1011102 450541)'::geometry limit 10;
SELECT track_id, dist FROM (
SELECT track_id, ST_DistanceCPA(tr,:qt) dist
FROM trajectories
ORDER BY tr |=| :qt
LIMIT 5
) foo;
SELECT *
FROM (
SELECT b.tlid, b.mtfcc,
b.geom <#> ST_GeomFromText('LINESTRING(746149 2948672,745954 2948576,
745787 2948499,745740 2948468,745712 2948438,
745690 2948384,745677 2948319)',2249) As b_dist,
ST_Distance(b.geom, ST_GeomFromText('LINESTRING(746149 2948672,745954 2948576,
745787 2948499,745740 2948468,745712 2948438,
745690 2948384,745677 2948319)',2249)) As act_dist
FROM bos_roads As b
ORDER BY b_dist, b.tlid
LIMIT 100) As foo
ORDER BY act_dist, tlid LIMIT 10;
WITH index_query AS (
SELECT ST_Distance(geom, 'SRID=3005;POINT(1011102 450541)'::geometry) as d,edabbr, vaabbr
FROM va2005
ORDER BY geom <<->> 'SRID=3005;POINT(1011102 450541)'::geometry LIMIT 100)
SELECT *
FROM index_query
ORDER BY d limit 10;
WITH index_query AS (
SELECT ST_Distance(geom, 'SRID=3005;POINT(1011102 450541)'::geometry) as d,edabbr, vaabbr
FROM va2005
ORDER BY geom <<#>> 'SRID=3005;POINT(1011102 450541)'::geometry LIMIT 100)
SELECT *
FROM index_query
ORDER BY d limit 10;

File diff suppressed because it is too large Load Diff

View File

@@ -3,7 +3,7 @@
# computed by SQLFluff when running the tests. Please run
# `python test/generate_parse_fixture_yml.py` to generate them after adding or
# altering SQL files.
_hash: c7f82e72dc37ec47d9013ba40c43b4e16f1722417eb887b0763fb164e4f3045e
_hash: 0fb5768779aab2ce85961a4826aba225f617cf326379763a3604ec13885bf57d
file:
statement:
select_statement:
@@ -58,7 +58,10 @@ file:
- naked_identifier: word
- dot: .
- naked_identifier: character_range
- binary_operator: '@>'
- json_operator:
at: '@'
comparison_operator:
raw_comparison_operator: '>'
- column_reference:
- naked_identifier: paragraph
- dot: .
@@ -68,7 +71,10 @@ file:
- naked_identifier: word
- dot: .
- naked_identifier: character_range
- binary_operator: <@
- json_operator:
comparison_operator:
raw_comparison_operator: <
at: '@'
- column_reference:
- naked_identifier: paragraph
- dot: .