Skip to content

Evaluation

This module is responsible for evaluating rules.

Evaluation

Evaluate a set of rules on a set of nodes.

Source code in src/dbt_score/evaluation.py 
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
class Evaluation:
    """Evaluate a set of rules on a set of nodes."""

    def __init__(
        self,
        rule_registry: RuleRegistry,
        manifest_loader: ManifestLoader,
        formatter: Formatter,
        scorer: Scorer,
        config: Config,
    ) -> None:
        """Create an Evaluation object.

        Args:
            rule_registry: A rule registry to access rules.
            manifest_loader: A manifest loader to access dbt metadata.
            formatter: A formatter to display results.
            scorer: A scorer to compute scores.
            config: A configuration.
        """
        self._rule_registry = rule_registry
        self._manifest_loader = manifest_loader
        self._formatter = formatter
        self._scorer = scorer
        self._config = config

        # For each evaluable, its results
        self.results: dict[Evaluable, EvaluableResultsType] = {}

        # For each evaluable, its computed score
        self.scores: dict[Evaluable, Score] = {}

        # The aggregated project score
        self.project_score: Score

    def evaluate(self) -> None:
        """Evaluate all rules."""
        rules = self._rule_registry.rules.values()

        for evaluable in chain(
            self._manifest_loader.models.values(),
            self._manifest_loader.sources.values(),
            self._manifest_loader.snapshots.values(),
            self._manifest_loader.exposures.values(),
            self._manifest_loader.seeds.values(),
        ):
            # type inference on elements from `chain` is wonky
            # and resolves to superclass HasColumnsMixin
            evaluable = cast(Evaluable, evaluable)
            self.results[evaluable] = {}
            for rule in rules:
                try:
                    if rule.should_evaluate(evaluable):
                        result = rule.evaluate(evaluable, **rule.config)
                        self.results[evaluable][rule.__class__] = result
                except Exception as e:
                    if self._config.debug:
                        traceback.print_exc()
                        pdb.post_mortem()
                    self.results[evaluable][rule.__class__] = e

            self.scores[evaluable] = self._scorer.score_evaluable(
                self.results[evaluable]
            )
            self._formatter.evaluable_evaluated(
                evaluable, self.results[evaluable], self.scores[evaluable]
            )

        # Compute score for project
        self.project_score = self._scorer.score_aggregate_evaluables(
            list(self.scores.values())
        )

        # Add null check before calling project_evaluated
        if (
            self._manifest_loader.models
            or self._manifest_loader.sources
            or self._manifest_loader.snapshots
            or self._manifest_loader.exposures
            or self._manifest_loader.seeds
        ):
            self._formatter.project_evaluated(self.project_score)

__init__(rule_registry, manifest_loader, formatter, scorer, config)

Create an Evaluation object.

Parameters:

Name Type Description Default
rule_registry RuleRegistry

A rule registry to access rules.

 required
manifest_loader ManifestLoader

A manifest loader to access dbt metadata.

 required
formatter Formatter

A formatter to display results.

 required
scorer Scorer

A scorer to compute scores.

 required
config Config

A configuration.

 required
Source code in src/dbt_score/evaluation.py 
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
def __init__(
    self,
    rule_registry: RuleRegistry,
    manifest_loader: ManifestLoader,
    formatter: Formatter,
    scorer: Scorer,
    config: Config,
) -> None:
    """Create an Evaluation object.

    Args:
        rule_registry: A rule registry to access rules.
        manifest_loader: A manifest loader to access dbt metadata.
        formatter: A formatter to display results.
        scorer: A scorer to compute scores.
        config: A configuration.
    """
    self._rule_registry = rule_registry
    self._manifest_loader = manifest_loader
    self._formatter = formatter
    self._scorer = scorer
    self._config = config

    # For each evaluable, its results
    self.results: dict[Evaluable, EvaluableResultsType] = {}

    # For each evaluable, its computed score
    self.scores: dict[Evaluable, Score] = {}

    # The aggregated project score
    self.project_score: Score

evaluate()

Evaluate all rules.

Source code in src/dbt_score/evaluation.py 
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
def evaluate(self) -> None:
    """Evaluate all rules."""
    rules = self._rule_registry.rules.values()

    for evaluable in chain(
        self._manifest_loader.models.values(),
        self._manifest_loader.sources.values(),
        self._manifest_loader.snapshots.values(),
        self._manifest_loader.exposures.values(),
        self._manifest_loader.seeds.values(),
    ):
        # type inference on elements from `chain` is wonky
        # and resolves to superclass HasColumnsMixin
        evaluable = cast(Evaluable, evaluable)
        self.results[evaluable] = {}
        for rule in rules:
            try:
                if rule.should_evaluate(evaluable):
                    result = rule.evaluate(evaluable, **rule.config)
                    self.results[evaluable][rule.__class__] = result
            except Exception as e:
                if self._config.debug:
                    traceback.print_exc()
                    pdb.post_mortem()
                self.results[evaluable][rule.__class__] = e

        self.scores[evaluable] = self._scorer.score_evaluable(
            self.results[evaluable]
        )
        self._formatter.evaluable_evaluated(
            evaluable, self.results[evaluable], self.scores[evaluable]
        )

    # Compute score for project
    self.project_score = self._scorer.score_aggregate_evaluables(
        list(self.scores.values())
    )

    # Add null check before calling project_evaluated
    if (
        self._manifest_loader.models
        or self._manifest_loader.sources
        or self._manifest_loader.snapshots
        or self._manifest_loader.exposures
        or self._manifest_loader.seeds
    ):
        self._formatter.project_evaluated(self.project_score)