Merge pull request #45 from Snowflake-Labs/release/v2.5.0

Release/v2.5.0 
  Added support for dynamic table object type
  Fixed error log for unsupported object type

CHANGELOG.md

@@ -1,6 +1,9 @@
 # DLSync Changelog
 
 This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+## [2.5.0] - 2025-11-06
+### Added
+- Added support for Dynamic Table object type
 ## [2.4.4] - 2025-10-27
 ### Fixed
 - Fixed issue in create script with quoted object names 

README.md

@@ -91,7 +91,7 @@ Inside this directory create a directory structure like:
 Where 
 - **database_name_*:** is the database name of your project, 
 - **schema_name_*:** are schemas inside the database, 
-- **object_type:** is type of the object only 1 of the following (VIEWS, FUNCTIONS, PROCEDURES, FILE_FORMATS, TABLES, SEQUENCES, STAGES, STREAMS, TASKS, STREAMLITS, PIPES, ALERTS)
+- **object_type:** is type of the object only 1 of the following (VIEWS, FUNCTIONS, PROCEDURES, FILE_FORMATS, TABLES, SEQUENCES, STAGES, STREAMS, TASKS, STREAMLITS, PIPES, ALERTS, DYNAMIC_TABLES),
 - **object_name_*.sql:** are individual database object scripts.
 - **config.yml:** is a configuration file used to configure DLSync behavior.
 - **parameter-[profile-*].properties:** is parameter to value map file. This is going to be used by corresponding individual instances of your database.
@@ -120,7 +120,7 @@ eg: view named SAMPLE_VIEW can have the following SQL statement in the `SAMPLE_V
 create or replace view ${MY_DB}.{MY_SCHEMA}.SAMPLE_VIEW as select * from ${MY_DB}.{MY_SECOND_SCHEMA}.MY_TABLE;
 ```
 #### 2. Migration Script
-This type of script is used for object types of TABLES, SEQUENCES, STAGES, STREAMS, TASKS and ALERTS.
+This type of script is used for object types of TABLES, SEQUENCES, STAGES, STREAMS, TASKS, ALERTS and DYNAMIC_TABLES.
 Here the script is treated as migration that will be applied to the object sequentially based on the version number. 
 This type of script contains 1 or more migration versions. One migration versions contains version number, author(optional), content (DDL or DML SQL statement) , rollback statement(optional) and verify statement(optional).
 Each migration version is immutable i.e Once the version is deployed you can not change the code of this version. Only you can add new versions.

example_scripts/main/example_db/MAIN_SCHEMA/DYNAMIC_TABLES/DAILY_ORDERS.SQL

@@ -0,0 +1,14 @@
+---version: 0, author: DlSync
+CREATE OR REPLACE DYNAMIC TABLE ${EXAMPLE_DB}.${MAIN_SCHEMA}.DAILY_ORDERS
+    TARGET_LAG = '30 minutes'
+    WAREHOUSE = ${MY_WAREHOUSE}
+AS
+    SELECT
+        PRODUCT_ID,
+        SUM(QUANTITY) AS TOTAL_QUANTITY,
+        ORDER_DATE::DATE AS ORDER_DATE,
+        COUNT(*) AS ORDER_COUNT
+    FROM ${MAIN_SCHEMA}.ORDERS
+    GROUP BY ORDER_DATE::DATE, PRODUCT_ID;
+---rollback: DROP DYNAMIC TABLE IF EXISTS ${EXAMPLE_DB}.${MAIN_SCHEMA}.DAILY_ORDERS;
+---verify: SELECT * FROM ${EXAMPLE_DB}.${MAIN_SCHEMA}.DAILY_ORDERS LIMIT 1;

gradle.properties

@@ -1 +1 @@
-releaseVersion=2.4.4
+releaseVersion=2.5.0

src/main/java/com/snowflake/dlsync/models/ScriptObjectType.java

@@ -1,7 +1,7 @@
 package com.snowflake.dlsync.models;
 
 public enum ScriptObjectType {
-    VIEWS("VIEW"),FUNCTIONS("FUNCTION"),PROCEDURES("PROCEDURE"),FILE_FORMATS("FILE FORMAT"),TABLES("TABLE"),STREAMS("STREAM"),SEQUENCES("SEQUENCE"),STAGES("STAGE"),TASKS("TASK"),STREAMLITS("STREAMLIT"),PIPES("PIPE"),ALERTS("ALERT");
+    VIEWS("VIEW"),FUNCTIONS("FUNCTION"),PROCEDURES("PROCEDURE"),FILE_FORMATS("FILE FORMAT"),TABLES("TABLE"),STREAMS("STREAM"),SEQUENCES("SEQUENCE"),STAGES("STAGE"),TASKS("TASK"),STREAMLITS("STREAMLIT"),PIPES("PIPE"),ALERTS("ALERT"),DYNAMIC_TABLES("DYNAMIC TABLE");
 
     private final String singular;
     private ScriptObjectType(String type) {
@@ -21,6 +21,7 @@ public enum ScriptObjectType {
             case STAGES:
             case TASKS:
             case ALERTS:
+            case DYNAMIC_TABLES:
                 return true;
             default:
                 return false;

src/main/java/com/snowflake/dlsync/parser/SqlTokenizer.java

@@ -29,7 +29,7 @@ public class SqlTokenizer {
     private static final String IDENTIFIER_REGEX = "((?:\\\"[^\"]+\\\"\\.)|(?:[{}$a-zA-Z0-9_]+\\.))?((?:\\\"[^\"]+\\\"\\.)|(?:[{}$a-zA-Z0-9_]+\\.))?(?i)";
     private static final String MIGRATION_REGEX = VERSION_REGEX + AUTHOR_REGEX + CONTENT_REGEX + ROLL_BACK_REGEX + VERIFY_REGEX;
 
-    private static final String DDL_REGEX = ";\\n+(CREATE\\s+OR\\s+REPLACE\\s+(TRANSIENT\\s|HYBRID\\s|SECURE\\s)?(?<type>FILE FORMAT|\\w+)\\s+(?<name>[\\\"\\w.]+)([\\s\\S]+?)(?=(;\\nCREATE\\s+)|(;$)))";
+    private static final String DDL_REGEX = ";\\n+(CREATE\\s+OR\\s+REPLACE\\s+(TRANSIENT\\s|HYBRID\\s|SECURE\\s)?(?<type>DYNAMIC TABLE|FILE FORMAT|VIEW|FUNCTION|PROCEDURE|TABLE|STREAM|SEQUENCE|STAGE|TASK|STREAMLIT|PIPE|ALERT|\\w+)\\s+(?<name>[\\\"\\w.]+)([\\s\\S]+?)(?=(;\\nCREATE\\s+)|(;$)))";
 
     private static final String STRING_LITERAL_REGEX = "(?<!as\\s{1,5})'([^'\\\\]*(?:\\\\.[^'\\\\]*)*(?:''[^'\\\\]*)*)'";
 
@@ -64,7 +64,12 @@ public class SqlTokenizer {
 
     public static Set<Script> parseScript(String filePath, String name, String scriptType, String content) {
         String objectName = SqlTokenizer.extractObjectName(name, content);
-        ScriptObjectType objectType = ScriptObjectType.valueOf(scriptType);
+        Optional<ScriptObjectType> optionalObjectType = Arrays.stream(ScriptObjectType.values()).filter( type -> type.toString().equalsIgnoreCase(scriptType)).findFirst();
+        if(!optionalObjectType.isPresent()) {
+            log.error("Unsupported object type: {} found!", scriptType, name);
+            throw new RuntimeException("Unknown script type of directory: " + scriptType);
+        }
+        ScriptObjectType objectType = optionalObjectType.get();
         String fullIdentifier = SqlTokenizer.getFirstFullIdentifier(objectName, content);
         if(fullIdentifier == null || fullIdentifier.isEmpty()) {
             log.error("Error reading script: {}, name and content mismatch", name);
@@ -245,6 +250,10 @@ public class SqlTokenizer {
         while(matcher.find()) {
             String content = matcher.group(1) + ";";
             String type = matcher.group("type");
+            if(type == null) {
+                log.error("Unable to parse object type from DDL: {}", content);
+                throw new RuntimeException("Unable to parse object type from DDL.");
+            }
             ScriptObjectType objectType = Arrays.stream(ScriptObjectType.values())
                     .filter(ot -> ot.getSingular().equalsIgnoreCase(type))
                     .collect(Collectors.toList()).get(0);

src/test/java/com/snowflake/dlsync/parser/SqlTokenizerTest.java

@@ -554,7 +554,51 @@ class SqlTokenizerTest {
         assertEquals(ScriptObjectType.PIPES, script.getObjectType(), "Object type should be PIPES");
         assertEquals(content, script.getContent(), "Script content should match the input content");
     }
-    
 
+    @Test
+    void parseScriptTypeDynamicTable() {
+        String filePath = "db_scripts/db1/schema1/DYNAMIC_TABLES/DYNAMIC_TABLE1.SQL";
+        String name = "DYNAMIC_TABLE1.SQL";
+        String scriptType = "DYNAMIC_TABLES";
+        String content = "---version: 0, author: dlsync \n" +
+                "CREATE OR REPLACE DYNAMIC TABLE db1.schema1.DYNAMIC_TABLE1\n" +
+                "  TARGET_LAG = '1 minute'\n" +
+                "  WAREHOUSE = 'my_warehouse'\n" +
+                "  AS SELECT id, name, COUNT(*) as count FROM db1.schema1.source_table GROUP BY id, name;\n" +
+                "---rollback: drop dynamic table db1.schema1.DYNAMIC_TABLE1;\n" +
+                "---verify: select * from db1.schema1.DYNAMIC_TABLE1;";
+
+        String expected_rollback = "drop dynamic table db1.schema1.DYNAMIC_TABLE1;";
+        String expected_verify = "select * from db1.schema1.DYNAMIC_TABLE1;";
+        Set<Script> scripts = SqlTokenizer.parseScript(filePath, name, scriptType, content);
+
+        assertNotNull(scripts, "Scripts should not be null");
+        assertEquals(1, scripts.size(), "There should be exactly one script parsed");
+
+        MigrationScript script = (MigrationScript) scripts.iterator().next();
+        assertEquals(0, script.getVersion(), "Version should be 0");
+        assertEquals(expected_rollback, script.getRollback(), "Rollback should match the input content");
+        assertEquals(expected_verify, script.getVerify(), "Verify should match the input content");
+        assertEquals("DYNAMIC_TABLE1", script.getObjectName(), "Object name should be DYNAMIC_TABLE1");
+        assertEquals("db1".toUpperCase(), script.getDatabaseName(), "Database name should be db1");
+        assertEquals("schema1".toUpperCase(), script.getSchemaName(), "Schema name should be schema1");
+        assertEquals(ScriptObjectType.DYNAMIC_TABLES, script.getObjectType(), "Object type should be DYNAMIC_TABLES");
+        assertEquals(content, script.getContent(), "Script content should match the input content");
+    }
+
+    @Test
+    void parseScriptUnsupportedObjectType() {
+        String filePath = "db_scripts/db1/schema1/UNKNOWN/OBJECT1.SQL";
+        String name = "OBJECT1.SQL";
+        String scriptType = "UNKNOWN_TYPE";
+        String content = "CREATE OR REPLACE UNKNOWN db1.schema1.OBJECT1;";
+
+        RuntimeException exception = assertThrows(RuntimeException.class, () -> {
+            SqlTokenizer.parseScript(filePath, name, scriptType, content);
+        }, "Should throw RuntimeException for unsupported object type");
+
+        assertEquals("Unknown script type of directory: UNKNOWN_TYPE", exception.getMessage(),
+                "Exception message should indicate unknown script type");
+    }
 
 }