From 0e26263ed4e25b044e587adac889c92ebb82e615 Mon Sep 17 00:00:00 2001 From: Steve Haines Date: Fri, 18 Apr 2025 22:56:06 +0100 Subject: [PATCH] add support for pipes --- .gitignore | 5 +++- .../example_db/MAIN_SCHEMA/PIPES/pipe.sql | 9 ++++++ pom.xml | 17 +++++++---- .../com/snowflake/dlsync/doa/ScriptRepo.java | 7 ++++- .../dlsync/models/ScriptObjectType.java | 2 +- .../dlsync/parser/SqlTokenizerTest.java | 30 +++++++++++++++++-- 6 files changed, 60 insertions(+), 10 deletions(-) create mode 100644 example_scripts/main/example_db/MAIN_SCHEMA/PIPES/pipe.sql diff --git a/.gitignore b/.gitignore index 909d2f3..a99adc4 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,7 @@ log/ target/ .env Jenkinsfile-app -out/ \ No newline at end of file +out/ +.vscode/ +Snowflake/ +*.p8 \ No newline at end of file diff --git a/example_scripts/main/example_db/MAIN_SCHEMA/PIPES/pipe.sql b/example_scripts/main/example_db/MAIN_SCHEMA/PIPES/pipe.sql new file mode 100644 index 0000000..f4f4061 --- /dev/null +++ b/example_scripts/main/example_db/MAIN_SCHEMA/PIPES/pipe.sql @@ -0,0 +1,9 @@ +create or replace pipe ${EXAMPLE_DB}.${MAIN_SCHEMA}.PIPE_STG_FINANCE_DATA +auto_ingest=true as +copy into ${EXAMPLE_DB}.${MAIN_SCHEMA}.STG_FINANCE_DATA ( + BRAND_CODE, + GROSS, + NET, + TAX +) +from @${EXAMPLE_DB}.${MAIN_SCHEMA}.STAGE_DEV_S3 (file_format => ${EXAMPLE_DB}.${MAIN_SCHEMA}.FF_S3_CSV, PATTERN => "FINANCE/\d{4}/\d{2}/\d{2}/FINANCEDATA.csv"); \ No newline at end of file diff --git a/pom.xml b/pom.xml index 3bca139..b209e4d 100644 --- a/pom.xml +++ b/pom.xml @@ -9,8 +9,6 @@ 2.0-SNAPSHOT - 11 - 11 @@ -50,7 +48,7 @@ org.projectlombok lombok 1.18.24 - provided + compile commons-cli @@ -58,8 +56,8 @@ 1.9.0 - antlr4-runtime - org.antlr + org.antlr + antlr4-runtime 4.13.2 @@ -80,6 +78,15 @@ + + org.apache.maven.plugins + maven-compiler-plugin + 3.11.0 + + 11 + + + org.apache.maven.plugins maven-surefire-plugin diff --git a/src/main/java/com/snowflake/dlsync/doa/ScriptRepo.java b/src/main/java/com/snowflake/dlsync/doa/ScriptRepo.java index 5b608d3..3a09f78 100644 --- a/src/main/java/com/snowflake/dlsync/doa/ScriptRepo.java +++ b/src/main/java/com/snowflake/dlsync/doa/ScriptRepo.java @@ -210,7 +210,12 @@ public class ScriptRepo { try { connection.setAutoCommit(false); if(!onlyHashes) { - statement.executeUpdate(script.getContent()); + if(script.getObjectType().getSingular() == "PIPE"){ + statement.execute(script.getContent()); + } + else{ + statement.executeUpdate(script.getContent()); + } log.debug("Creating object using the SQL: {}", script.getContent()); } updateScriptHash(script); diff --git a/src/main/java/com/snowflake/dlsync/models/ScriptObjectType.java b/src/main/java/com/snowflake/dlsync/models/ScriptObjectType.java index 8fe0e7c..5998415 100644 --- a/src/main/java/com/snowflake/dlsync/models/ScriptObjectType.java +++ b/src/main/java/com/snowflake/dlsync/models/ScriptObjectType.java @@ -1,7 +1,7 @@ package com.snowflake.dlsync.models; public enum ScriptObjectType { - VIEWS("VIEW"),FUNCTIONS("FUNCTION"),PROCEDURES("PROCEDURE"),FILE_FORMATS("FILE FORMAT"),TABLES("TABLE"),STREAMS("STREAM"),SEQUENCES("SEQUENCE"),STAGES("STAGE"),TASKS("TASK"),STREAMLITS("STREAMLIT"); + VIEWS("VIEW"),FUNCTIONS("FUNCTION"),PROCEDURES("PROCEDURE"),FILE_FORMATS("FILE FORMAT"),TABLES("TABLE"),STREAMS("STREAM"),SEQUENCES("SEQUENCE"),STAGES("STAGE"),TASKS("TASK"),STREAMLITS("STREAMLIT"),PIPES("PIPE"); private final String singular; private ScriptObjectType(String type) { diff --git a/src/test/java/com/snowflake/dlsync/parser/SqlTokenizerTest.java b/src/test/java/com/snowflake/dlsync/parser/SqlTokenizerTest.java index c7d90c8..6b3a834 100644 --- a/src/test/java/com/snowflake/dlsync/parser/SqlTokenizerTest.java +++ b/src/test/java/com/snowflake/dlsync/parser/SqlTokenizerTest.java @@ -505,13 +505,39 @@ class SqlTokenizerTest { assertEquals(1, scripts.size(), "There should be exactly one script parsed"); Script script = scripts.iterator().next(); - assertEquals("STREAMLIT1", script.getObjectName(), "Object name should be VIEW1"); + assertEquals("STREAMLIT1", script.getObjectName(), "Object name should be STREAMLIT1"); assertEquals("db1".toUpperCase(), script.getDatabaseName(), "Database name should be db1"); assertEquals("schema1".toUpperCase(), script.getSchemaName(), "Schema name should be schema1"); - assertEquals(ScriptObjectType.STREAMLITS, script.getObjectType(), "Object type should be VIEWS"); + assertEquals(ScriptObjectType.STREAMLITS, script.getObjectType(), "Object type should be STREAMLITS"); assertEquals(content, script.getContent(), "Script content should match the input content"); } + @Test + void parseScriptTypePipe() { + String filePath = "db_scripts/db1/schema1/PIPES/PIPE1.SQL"; + String name = "PIPE1.SQL"; + String scriptType = "PIPES"; + String content = "CREATE OR REPLACE PIPE db1.schema1.PIPE1\n" + + " AUTO_INGEST = TRUE\n" + + " COMMENT = 'Loads data automatically from stage'\n" + + " AS COPY INTO db1.schema1.my_table\n" + + " FROM @my_stage/file_prefix\n" + + " FILE_FORMAT = (TYPE = 'CSV');"; + + Set