Compare commits

...

60 Commits

Author SHA1 Message Date
Michelle Ark
a8b1bc9745 Merge branch 'main' into 7444/spike-generate-macro-resolution 2023-05-03 16:35:49 -04:00
Michelle Ark
e38e0c8813 find_default_generate_macro_by_name, find_package_generate_macros_by_name macros 2023-05-03 16:30:05 -04:00
Gerda Shank
7471b2f349 develop multi project test case 2023-05-03 09:58:37 -04:00
Gerda Shank
a0611a47fd Create fixture to create minimal alternate project (just models) 2023-05-02 14:05:26 -04:00
Gerda Shank
f12e9b6e50 Merge branch 'main' into ct-2327-model_publication 2023-05-01 14:21:06 -04:00
Gerda Shank
f705d26759 Code review cleanup 2023-05-01 10:21:27 -04:00
Gerda Shank
6369410694 Merge branch 'main' into ct-2327-model_publication 2023-04-28 16:47:56 -04:00
Gerda Shank
552b623952 Make write_publication_artifact a function in parser/manifest.py 2023-04-27 15:15:02 -04:00
Gerda Shank
b81157fdac Change PublicationArtifactChanged message to debug level 2023-04-27 13:24:48 -04:00
Gerda Shank
3fc17fe059 Merge branch 'main' into ct-2327-model_publication 2023-04-27 12:34:03 -04:00
Gerda Shank
97f1fc9385 Remove create_from_relation_name, call create_from_node instead 2023-04-27 12:01:23 -04:00
Gerda Shank
1589c754c6 Include external publication dependencies in publication artifact dependencies 2023-04-27 11:46:16 -04:00
Gerda Shank
a256083a36 Remove "environment" from ProjectDependency. Add
database/schema/identifier to PublicModel. Update TargetNotFound
exception.
2023-04-27 11:37:04 -04:00
Gerda Shank
fce5c070ca Move some file name constants to core/dbt/constants.py 2023-04-27 11:26:44 -04:00
Gerda Shank
2aad937740 Fix test_manifest.py 2023-04-26 16:58:16 -04:00
Gerda Shank
325415478c Rename some dependencies 2023-04-26 16:40:23 -04:00
Gerda Shank
acfae95cf6 Add public_nodes to flat_graph 2023-04-26 16:27:03 -04:00
Gerda Shank
0425c3bcf6 Remove duplicate writing of manifest.json 2023-04-26 14:35:49 -04:00
Gerda Shank
4fdfb2b475 cleanup depends_on_public_nodes; move call to rebuild_ref_lookup; tweak
if in ref lookup
2023-04-26 14:23:45 -04:00
Gerda Shank
f286af2fdb get quote character from class, quoting from publication artifact 2023-04-25 19:57:43 -04:00
Gerda Shank
6b01773ca3 Some comments and minor cleanup 2023-04-25 17:23:28 -04:00
Gerda Shank
2e938e3317 refactor relation_from_relation_name 2023-04-25 17:05:39 -04:00
Gerda Shank
dac9fa8ce8 Merge branch 'main' into ct-2327-model_publication 2023-04-25 14:41:48 -04:00
Gerda Shank
bac92eadf2 Remove duplicate nodes from manifest 2023-04-24 15:52:48 -04:00
Gerda Shank
41364be1d8 Merge branch 'main' into ct-2327-model_publication 2023-04-24 15:47:58 -04:00
Gerda Shank
719ab9a52b Add some logging events 2023-04-24 15:02:52 -04:00
Gerda Shank
34b813a93b Add another test 2023-04-24 14:27:29 -04:00
Gerda Shank
9e2284c1dd Handle only changed publication artifacts 2023-04-24 14:12:10 -04:00
Gerda Shank
e9d5a4c9aa Handle removing references for re-processing if publication has changed 2023-04-21 16:24:48 -04:00
Gerda Shank
3f54bb0ea6 Update another instance check of PublicModel 2023-04-21 11:31:04 -04:00
Gerda Shank
c4db8c0857 Fix another isinstance of public model 2023-04-20 12:58:55 -04:00
Gerda Shank
a5429cc389 Merge branch 'main' into ct-2327-model_publication 2023-04-20 12:56:40 -04:00
Gerda Shank
fcd545b3b6 Changie 2023-04-20 12:48:04 -04:00
Gerda Shank
3054aed817 change detection of PublicModel for >= python3.10 2023-04-20 12:45:16 -04:00
Gerda Shank
942f7120eb Fix is_latest_version and add some comments 2023-04-19 16:24:17 -04:00
Gerda Shank
a43327df80 Remove ModelDependsOn 2023-04-19 11:30:40 -04:00
Gerda Shank
5bd828e396 Store dependencies in publication artifact 2023-04-19 10:13:37 -04:00
Gerda Shank
f7202ae894 Split out PublicationArtifact and PublicationConfig, store public_models
separately
2023-04-18 15:58:15 -04:00
Gerda Shank
0dd963a34a Fix refs to manifest.json version, typo in Dependencies 2023-04-18 14:33:02 -04:00
Gerda Shank
22cb844228 Merge branch 'main' into ct-2327-model_publication 2023-04-18 14:04:21 -04:00
Gerda Shank
fb12f7cd0b re-comment line in test_previous_version_state.py 2023-04-18 12:03:13 -04:00
Gerda Shank
c3c2940568 Bump manifest version and fix tests, use ModelDependsOn 2023-04-17 17:03:09 -04:00
Gerda Shank
17542fb587 Add public nodes to parent and child maps 2023-04-17 13:30:00 -04:00
Gerda Shank
c96c2a2b1d Test working with public_node 2023-04-17 12:40:40 -04:00
Gerda Shank
bd652a7cba relation from relation_name. Not tested. 2023-04-17 12:28:15 -04:00
Gerda Shank
babaf6a7d5 rename external_nodes to public_nodes 2023-04-17 11:37:33 -04:00
Gerda Shank
22d6e851e3 Merge branch 'main' into ct-2327-model_publication 2023-04-17 10:02:15 -04:00
Gerda Shank
f8a7bfdf8e add "external_nodes" dictionary, save external ref unique_ids to it 2023-04-14 16:27:50 -04:00
Gerda Shank
6a9c06dd66 resolve_ref working 2023-04-14 15:44:31 -04:00
Gerda Shank
800934946e Add "external_nodes" and populate ref_lookup 2023-04-14 15:32:26 -04:00
Gerda Shank
86045ca67a add "name" and "package_name" to PublicModel, some changes to ref_lookup 2023-04-14 15:22:17 -04:00
Gerda Shank
ede647ac55 Add unique_id and a couple of properties to PublicModel 2023-04-14 13:04:05 -04:00
Gerda Shank
8dcc2b5cac change test to use parse and get manifest 2023-04-14 11:22:55 -04:00
Gerda Shank
6bb6e5767b Add version to PublicModel for now 2023-04-14 11:12:45 -04:00
Gerda Shank
77d2a5ba00 use is_latest_version 2023-04-14 10:50:45 -04:00
Gerda Shank
9add2a61f2 Load dependencies.yml and the corresponding publication file 2023-04-13 14:41:55 -04:00
Gerda Shank
8ba476abe3 Move test_publication into multi_project dir 2023-04-13 13:11:41 -04:00
Gerda Shank
7ffadafe10 Fix test 2023-04-12 11:56:26 -04:00
Gerda Shank
6d511ebe06 fix merge error 2023-04-12 11:49:55 -04:00
Gerda Shank
0daea9fb5a Create publication.py, various Publication classes, Dependency class 2023-04-12 11:46:41 -04:00
2 changed files with 45 additions and 7 deletions

View File

@@ -568,11 +568,11 @@ class MacroMethods:
return candidates.last()
def find_generate_macro_by_name(
def find_default_generate_macro_by_name(
self, component: str, root_project_name: str
) -> Optional[Macro]:
"""
The `generate_X_name` macros are similar to regular ones, but ignore
The default `generate_X_name` macros are similar to regular ones, and ignore
imported packages.
- if there is a `generate_{component}_name` macro in the root
project, return it
@@ -591,6 +591,21 @@ class MacroMethods:
)
return candidates.last()
def find_package_generate_macros_by_name(
self, component: str, root_project_name: str
) -> List[Macro]:
"""
Return any `generate_X_name` macros defined in imported packages
"""
def filter(candidate: MacroCandidate) -> bool:
return candidate.locality == Locality.Imported
candidates: CandidateList = self._find_macros_by_name(
name=f"generate_{component}_name", root_project_name=root_project_name, filter=filter
)
return [candidate.macro for candidate in candidates]
def _find_macros_by_name(
self,
name: str,

View File

@@ -71,20 +71,43 @@ class Parser(BaseParser[FinalValue], Generic[FinalValue]):
class RelationUpdate:
def __init__(self, config: RuntimeConfig, manifest: Manifest, component: str) -> None:
macro = manifest.find_generate_macro_by_name(
default_macro = manifest.find_default_generate_macro_by_name(
component=component,
root_project_name=config.project_name,
)
if macro is None:
package_macros = manifest.find_package_generate_macros_by_name(
component=component,
root_project_name=config.project_name,
)
if default_macro is None:
raise DbtInternalError(f"No macro with name generate_{component}_name found")
root_context = generate_generate_name_macro_context(macro, config, manifest)
self.updater = MacroGenerator(macro, root_context)
default_macro_context = generate_generate_name_macro_context(
default_macro, config, manifest
)
self.default_updater = MacroGenerator(default_macro, default_macro_context)
package_updaters = {}
for package_macro in package_macros:
imported_macro_context = generate_generate_name_macro_context(
package_macro, config, manifest
)
package_updaters[package_macro.package_name] = MacroGenerator(
package_macro, imported_macro_context
)
self.package_updaters = package_updaters
self.component = component
def __call__(self, parsed_node: Any, config_dict: Dict[str, Any]) -> None:
override = config_dict.get(self.component)
new_value = self.updater(override, parsed_node)
if parsed_node.package_name in self.package_updaters:
new_value = self.package_updaters[parsed_node.package_name](override, parsed_node)
else:
new_value = self.default_updater(override, parsed_node)
if isinstance(new_value, str):
new_value = new_value.strip()
setattr(parsed_node, self.component, new_value)