repo-mirrors/dlsync
1
0
Fork 0
mirror of https://github.com/Snowflake-Labs/dlsync.git synced 2025-12-18 00:51:27 +00:00
Code Actions Packages Releases Activity

Merge pull request #47 from Snowflake-Labs/release/v2.6.0

Browse Source 
Release/v2.6.0
This commit is contained in:
Ytbarek Hailu
2025-11-24 13:17:40 -08:00
committed by GitHub
parent 0384c28f0d 63b01e3131
commit 9a86b25d8d
6 changed files with 57 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG.md
View File

@@ -1,6 +1,10 @@
# DLSync Changelog
This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [2.6.0] - 2025-11-24
### Added
- Added support Masking Policy object type
- Fixed log message unsupported object types in create script
## [2.5.0] - 2025-11-06
### Added
- Added support for Dynamic Table object type

4
README.md
View File

@@ -91,7 +91,7 @@ Inside this directory create a directory structure like:
Where
- **database_name_*:** is the database name of your project,
- **schema_name_*:** are schemas inside the database,
- **object_type:** is type of the object only 1 of the following (VIEWS, FUNCTIONS, PROCEDURES, FILE_FORMATS, TABLES, SEQUENCES, STAGES, STREAMS, TASKS, STREAMLITS, PIPES, ALERTS, DYNAMIC_TABLES),
- **object_type:** is type of the object only 1 of the following (VIEWS, FUNCTIONS, PROCEDURES, FILE_FORMATS, TABLES, SEQUENCES, STAGES, STREAMS, TASKS, STREAMLITS, PIPES, ALERTS, DYNAMIC_TABLES, MASKING_POLICIES),
- **object_name_*.sql:** are individual database object scripts.
- **config.yml:** is a configuration file used to configure DLSync behavior.
- **parameter-[profile-*].properties:** is parameter to value map file. This is going to be used by corresponding individual instances of your database.
@@ -104,7 +104,7 @@ Each object will have a single SQL to track the changes applied to the given obj
For example if you have a view named `SAMPLE_VIEW` in schema `MY_SCHEMA` in database `MY_DATABASE`, then the script file should be named `SAMPLE_VIEW.SQL` and should be placed in the directory `[scripts_root]/main/MY_DATABASE/MY_SCHEMA/VIEWS/SAMPLE_VIEW.SQL`.
The structure and content of the scripts will defer based on the type of script. This tool categorizes script in to 2 types named State script and Migration script.
#### 1. State Script
This type of script is used for object types of Views, UDF, Stored Procedure, File formats and Pipes.
This type of script is used for object types of VIEWS, UDF, PROCEDURES, FILE FORMATS, PIPES AND MASKING POLICIES.
In this type of script you define the current state(desired state) of the object.
When a change is made to the script, DLSync replaces the current object with the updated definition.
These types of scripts must always have `create or replace` statement. Every time you make a change to the script DLSync will replace the object with the new definition.

7
example_scripts/main/example_db/MAIN_SCHEMA/MASKING_POLICIES/EMAIL_MASK.SQL Normal file
View File

@@ -0,0 +1,7 @@
CREATE OR REPLACE MASKING POLICY ${EXAMPLE_DB}.${MAIN_SCHEMA}.EMAIL_MASK AS (val STRING)
RETURNS STRING ->
CASE
WHEN CURRENT_ROLE() IN ('ADMIN', 'ANALYST') THEN val
ELSE '***MASKED***'
END;

2
src/main/java/com/snowflake/dlsync/models/ScriptObjectType.java
View File

@@ -1,7 +1,7 @@
package com.snowflake.dlsync.models;
public enum ScriptObjectType {
VIEWS("VIEW"),FUNCTIONS("FUNCTION"),PROCEDURES("PROCEDURE"),FILE_FORMATS("FILE FORMAT"),TABLES("TABLE"),STREAMS("STREAM"),SEQUENCES("SEQUENCE"),STAGES("STAGE"),TASKS("TASK"),STREAMLITS("STREAMLIT"),PIPES("PIPE"),ALERTS("ALERT"),DYNAMIC_TABLES("DYNAMIC TABLE");
VIEWS("VIEW"),FUNCTIONS("FUNCTION"),PROCEDURES("PROCEDURE"),FILE_FORMATS("FILE FORMAT"),TABLES("TABLE"),STREAMS("STREAM"),SEQUENCES("SEQUENCE"),STAGES("STAGE"),TASKS("TASK"),STREAMLITS("STREAMLIT"),PIPES("PIPE"),ALERTS("ALERT"),DYNAMIC_TABLES("DYNAMIC TABLE"),MASKING_POLICIES("MASKING POLICY");
private final String singular;
private ScriptObjectType(String type) {

11
src/main/java/com/snowflake/dlsync/parser/SqlTokenizer.java
View File

@@ -29,7 +29,7 @@ public class SqlTokenizer {
private static final String IDENTIFIER_REGEX = "((?:\\\"[^\"]+\\\"\\.)|(?:[{}$a-zA-Z0-9_]+\\.))?((?:\\\"[^\"]+\\\"\\.)|(?:[{}$a-zA-Z0-9_]+\\.))?(?i)";
private static final String MIGRATION_REGEX = VERSION_REGEX + AUTHOR_REGEX + CONTENT_REGEX + ROLL_BACK_REGEX + VERIFY_REGEX;
private static final String DDL_REGEX = ";\\n+(CREATE\\s+OR\\s+REPLACE\\s+(TRANSIENT\\s|HYBRID\\s|SECURE\\s)?(?<type>DYNAMIC TABLE|FILE FORMAT|VIEW|FUNCTION|PROCEDURE|TABLE|STREAM|SEQUENCE|STAGE|TASK|STREAMLIT|PIPE|ALERT|\\w+)\\s+(?<name>[\\\"\\w.]+)([\\s\\S]+?)(?=(;\\nCREATE\\s+)|(;$)))";
private static final String DDL_REGEX = ";\\n+(CREATE\\s+OR\\s+REPLACE\\s+(TRANSIENT\\s|HYBRID\\s|SECURE\\s)?(?<type>DYNAMIC TABLE|FILE FORMAT|MASKING POLICY|VIEW|FUNCTION|PROCEDURE|TABLE|STREAM|SEQUENCE|STAGE|TASK|STREAMLIT|PIPE|ALERT|\\w+)\\s+(?<name>[\\\"\\w.]+)([\\s\\S]+?)(?=(;\\nCREATE\\s+)|(;$)))";
private static final String STRING_LITERAL_REGEX = "(?<!as\\s{1,5})'([^'\\\\]*(?:\\\\.[^'\\\\]*)*(?:''[^'\\\\]*)*)'";
@@ -254,9 +254,12 @@ public class SqlTokenizer {
log.error("Unable to parse object type from DDL: {}", content);
throw new RuntimeException("Unable to parse object type from DDL.");
}
ScriptObjectType objectType = Arrays.stream(ScriptObjectType.values())
.filter(ot -> ot.getSingular().equalsIgnoreCase(type))
.collect(Collectors.toList()).get(0);
Optional<ScriptObjectType> optionalObjectType = Arrays.stream(ScriptObjectType.values()).filter( ot -> ot.getSingular().equalsIgnoreCase(type)).findFirst();
if(!optionalObjectType.isPresent()) {
log.error("Unsupported object type: {} found in DDL!", type);
throw new RuntimeException("Unknown object type found in DDL: " + type);
}
ScriptObjectType objectType = optionalObjectType.get();
String fullObjectName = matcher.group("name");
String scriptObjectName = fullObjectName.split("\\.")[2];

36
src/test/java/com/snowflake/dlsync/parser/SqlTokenizerTest.java
View File

@@ -202,6 +202,8 @@ class SqlTokenizerTest {
"create or replace transient table db1.schema1.table2 (col1 varchar, col2 number);\n" +
"create or replace hybrid table db1.schema1.table3 (col1 varchar, col2 number);\n" +
"create or replace table db1.schema1.\"table4\" (col1 varchar, col2 number);\n" +
"create or replace dynamic table db1.schema1.dynamic_table1 (col1 varchar, col2 number)\n as SELECT id, name, COUNT(*) as count FROM db1.schema1.source_table GROUP BY id, name;\n" +
"create or replace masking policy db1.schema1.masking_policy1 as (val string) returns string -> case when current_role() in ('ANALYST_ROLE', 'PUBLIC') then val else '****' end;\n" +
"create or replace function db1.schema1.function1(arg1 varchar)\n" +
"RETURNS VARCHAR(16777216)\n" +
"LANGUAGE JAVASCRIPT\n" +
@@ -215,6 +217,8 @@ class SqlTokenizerTest {
ScriptFactory.getMigrationScript("db1", "schema1", ScriptObjectType.TABLES, "table2","create or replace transient table db1.schema1.table2 (col1 varchar, col2 number);"),
ScriptFactory.getMigrationScript("db1", "schema1", ScriptObjectType.TABLES, "table3","create or replace hybrid table db1.schema1.table3 (col1 varchar, col2 number);"),
ScriptFactory.getMigrationScript("db1", "schema1", ScriptObjectType.TABLES, "\"table4\"","create or replace table db1.schema1.\"table4\" (col1 varchar, col2 number);"),
ScriptFactory.getMigrationScript("db1", "schema1", ScriptObjectType.DYNAMIC_TABLES, "dynamic_table1","create or replace dynamic table db1.schema1.dynamic_table1 (col1 varchar, col2 number)\n as SELECT id, name, COUNT(*) as count FROM db1.schema1.source_table GROUP BY id, name;"),
ScriptFactory.getStateScript("db1", "schema1", ScriptObjectType.MASKING_POLICIES, "masking_policy1","create or replace masking policy db1.schema1.masking_policy1 as (val string) returns string -> case when current_role() in ('ANALYST_ROLE', 'PUBLIC') then val else '****' end;"),
ScriptFactory.getStateScript("db1", "schema1", ScriptObjectType.FUNCTIONS, "function1","create or replace function db1.schema1.function1(arg1 varchar)\n" +
"RETURNS VARCHAR(16777216)\n" +
"LANGUAGE JAVASCRIPT\n" +
@@ -586,6 +590,26 @@ class SqlTokenizerTest {
assertEquals(content, script.getContent(), "Script content should match the input content");
}
@Test
void parseScriptTypeMaskingPolicy() {
String filePath = "db_scripts/db1/schema1/MASKING_POLICIES/EMAIL_MASK.SQL";
String name = "EMAIL_MASK.SQL";
String scriptType = "MASKING_POLICIES";
String content = "CREATE OR REPLACE MASKING POLICY db1.schema1.EMAIL_MASK AS (val STRING) RETURNS STRING -> CASE WHEN CURRENT_ROLE() IN ('ADMIN') THEN val ELSE '***MASKED***' END;";
Set<Script> scripts = SqlTokenizer.parseScript(filePath, name, scriptType, content);
assertNotNull(scripts, "Scripts should not be null");
assertEquals(1, scripts.size(), "There should be exactly one script parsed");
Script script = scripts.iterator().next();
assertEquals("EMAIL_MASK", script.getObjectName(), "Object name should be EMAIL_MASK");
assertEquals("db1".toUpperCase(), script.getDatabaseName(), "Database name should be db1");
assertEquals("schema1".toUpperCase(), script.getSchemaName(), "Schema name should be schema1");
assertEquals(ScriptObjectType.MASKING_POLICIES, script.getObjectType(), "Object type should be MASKING_POLICIES");
assertEquals(content, script.getContent(), "Script content should match the input content");
}
@Test
void parseScriptUnsupportedObjectType() {
String filePath = "db_scripts/db1/schema1/UNKNOWN/OBJECT1.SQL";
@@ -601,4 +625,16 @@ class SqlTokenizerTest {
"Exception message should indicate unknown script type");
}
@Test
void parseDdlScriptUnsupportedObjectType() {
String ddl = "create or replace schema schema1;\n\nCREATE OR REPLACE UNKNOWN db1.schema1.OBJECT1;";
RuntimeException exception = assertThrows(RuntimeException.class, () -> {
SqlTokenizer.parseDdlScripts(ddl, "db1", "schema1");
}, "Should throw RuntimeException for unsupported object type");
assertEquals("Unknown object type found in DDL: UNKNOWN", exception.getMessage(),
"Exception message should indicate unsupported DDL statement");
}
}