rtdip · GBBBAS · Dec 3, 2024 · Dec 4, 2024 · Dec 4, 2024 · Dec 4, 2024
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -27,7 +27,7 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         python-version: ["3.9", "3.10", "3.11", "3.12"]
-        pyspark: ["3.3.0", "3.3.1", "3.3.2", "3.4.0", "3.4.1", "3.5.0", "3.5.1"]
+        pyspark: ["3.3.0", "3.3.1", "3.3.2", "3.4.0", "3.4.1", "3.5.0", "3.5.1"] # 3.5.2 does not work  with conda
         exclude:
           - pyspark: "3.5.1"
             python-version: "3.9"

diff --git a/.gitignore b/.gitignore
@@ -136,4 +136,7 @@ spark-warehouse/
 spark-checkpoints/
 
 # Delta Sharing
-config.share
+config.share
+
+# JetBrains
+.idea/
diff --git a/docs/blog/.authors.yml b/docs/blog/.authors.yml
@@ -24,4 +24,8 @@ authors:
   GBARAS:
     name: Amber Rigg
     description: Contributor
-    avatar: https://github.com/Amber-Rigg.png
+    avatar: https://github.com/Amber-Rigg.png
+  TUBCM:
+    name: Christian Munz
+    description: Contributor
+    avatar: https://github.com/chris-1187.png
diff --git a/docs/blog/images/agile.svg b/docs/blog/images/agile.svg
diff --git a/docs/blog/images/amos_mvi.png b/docs/blog/images/amos_mvi.png
diff --git a/docs/blog/images/amos_mvi_raw.png b/docs/blog/images/amos_mvi_raw.png
diff --git a/docs/blog/posts/enhancing_data_quality_amos.md b/docs/blog/posts/enhancing_data_quality_amos.md
@@ -0,0 +1,94 @@
+---
+date: 2025-02-05
+authors:
+  - TUBCM
+---
+
+# Enhancing Data Quality in Real-Time: Our Experience with RTDIP and the AMOS Project
+
+<center>
+
+![blog](../images/agile.svg){width=60%}
+<small>1</small>
+</center>
+
+Real-time data integration and preparation are crucial in today's data-driven world, especially when dealing with time series data from often distributed heterogeneous data sources. As data scientists often spend no less than 80%<sup>2</sup> of their time finding, integrating, and cleaning datasets, the importance of automated ingestion pipelines rises inevitably. Building such ingestion and integration frameworks can be challenging and can entail all sorts of technical debt like glue code, pipeline jungles, or dead code paths, which calls for precise conception and development of such systems. Modern software development approaches try to mitigate technical debts and enhance quality results by introducing and utilizing agile and more iterative methodologies, which are designed to foster rapid feedback and continuous progress.
+
+<!-- more -->
+
+As part of the Agile Methods and Open Source (AMOS) project, we had the unique opportunity to work in a SCRUM team consisting of students from TU Berlin and FAU Erlangen-Nürnberg, to build data quality measures for the RTDIP Ingestion Pipeline framework. With the goal of enhancing data quality, we got to work and built modular pipeline components that aim to help data scientists and engineers with data integration, data cleaning, and data preparation.
+
+But what does it mean to work in an agile framework? The Agile Manifesto is above all a set of guiding values, principles, ideals, and goals. The overarching goal is to gain performance and be most effective while adding business value. By prioritizing the right fundamentals like individuals and interactions, working software, customer collaboration, and responding to change, cross-functional teams can ship viable products easier and faster.
+
+How that worked out for us in building data quality measures? True to the motto "User stories drive everything," we got together with contributors from the RTDIP Team to hear about concepts, the end users' stake in the project, and the current state to get a grasp on the expectations we can set on ourselves. With that, we got to work and planned our first sprint, and soon, we got the idea of how agile implementation is here to point out deficiencies in our processes. Through regular team meetings, we fostered a culture of continuous feedback and testing, leveraging reviews and retrospectives to identify roadblocks and drive necessary changes that enhance the overall development process.
+
+## Enhancing Data Quality in RTDIP's Pipeline Framework
+
+Coming up with modular steps that enhance data quality was the initial and arguably most critical step to start off a successful development process. So the question was: what exactly do the terms data integration, data cleaning, and data preparation entail? To expand on the key parts of that, this is what we did to pour these aspects into RTDIP components.
+
+### Data Validation and Schema Alignment
+
+Data validation and schema alignment are critical for ensuring the reliability and usability of data, serving as a foundational step before implementing other quality measures. For the time series data at hand, we developed an InputValidator component to verify that incoming data adheres to predefined quality standards, including compliance with an expected schema, correct PySpark data types, and proper handling of null values, raising exceptions when inconsistencies are detected. Additionally, the component enforces schema integration, harmonizing data from multiple sources into a unified, predefined structure. To maintain a consistent and efficient workflow, we required all data quality components to inherit the validation functionality of the InputValidator.
+
+### Data Cleansing
+
+Data cleansing is a vital process in enhancing the quality of data within a data integration pipeline, ensuring consistency, reliability, and usability. We implemented functionalities such as duplicate detection, which identifies and removes redundant records to prevent skewed analysis, and flatline filters, which eliminate constant, non-informative data points. Interval and range filters are employed to validate the time series data against predefined temporal or value ranges, ensuring conformity with expected patterns. Additionally, a K-sigma anomaly detection component identifies outliers based on statistical deviations, enabling the isolation of erroneous or anomalous values. Together, these methods ensure the pipeline delivers high-quality, actionable data for downstream processes.
+
+### Missing Value Imputation
+
+With a dataset refined to exclude unwanted data points and accounting for potential sensor failures, the next step toward ensuring high-quality data is to address any missing values through imputation. The component we developed first identifies and flags missing values by leveraging PySpark’s capabilities in windowing and UDF operations. With these techniques, we are able to dynamically determine the expected interval for each sensor by analyzing historical data patterns within defined partitions. Spline interpolation allows us to estimate missing values in time series data, seamlessly filling gaps with plausible and mathematically derived substitutes. By doing so, data scientists can not only improve the consistency of integrated datasets but also prevent errors or biases in analytics and machine learning models.
+To actually show how this is realized with this new RTDIP component, let me show you a short example on how a few lines of code can enhance an exemplary time series load profile:
+```python
+from rtdip_sdk.pipelines.data_quality import MissingValueImputation
+from pyspark.sql import SparkSession
+import pandas as pd
+
+spark_session = SparkSession.builder.master("local[2]").appName("test").getOrCreate()
+
+source_df = pd.read_csv('./solar_energy_production_germany_April02.csv')
+incomplete_spark_df = spark_session.createDataFrame(vi_april_df, ['Value', 'EventTime', 'TagName', 'Status'])
+
+#Before Missing Value Imputation
+spark_df.show()
+
+#Execute RTDIP Pipeline component
+clean_df = MissingValueImputation(spark_session, df=incomplete_spark_df).filter_data()
+
+#After Missing Value Imputation
+clean_df.show()
+```
+To illustrate this visually, plotting the before-and-after DataFrames reveals that all gaps have been successfully filled with meaningful data.
+
+<center>
+
+![blog](../images/amos_mvi_raw.png){width=70%}
+
+![blog](../images/amos_mvi.png){width=70%}
+
+</center>
+
+
+### Normalization
+
+Normalization is a critical step in ensuring data quality within data integration pipelines with various sources. Techniques like mean normalization, min-max scaling, and z-score standardization help transform raw time series data into a consistent scale, eliminating biases caused by differing units or magnitudes across features. It enables fair comparisons between variables, accelerates algorithm convergence, and ensures that data from diverse sources aligns seamlessly, supporting possible downstream processes such as entity resolution, data augmentation, and machine learning. To offer a variety of use cases within the RTDIP pipeline, we implemented normalization techniques like mean normalization, min-max scaling, and z-score standardization as well as their respective denormalization methods.
+
+### Data Monitoring
+
+Data monitoring is another aspect of enhancing data quality within the RTDIP pipeline, ensuring the reliability and consistency of incoming data streams. Techniques such as flatline detection identify periods of unchanging values, which may indicate sensor malfunctions or stale data. Missing data identification leverages predefined intervals or historical patterns to detect and flag gaps, enabling proactive resolution. By continuously monitoring for these anomalies, the pipeline maintains high data integrity, supporting accurate analysis for inconsistencies.
+
+### Data Prediction
+
+Forecasting based on historical data patterns is essential for making informed decisions on a business level. Linear Regression is a simple yet powerful approach for predicting continuous outcomes by establishing a relationship between input features and the target variable. However, for time series data, the ARIMA (Autoregressive Integrated Moving Average) model is often preferred due to its ability to model temporal dependencies and trends in the data. The ARIMA model combines autoregressive (AR) and moving average (MA) components, along with differencing to stabilize the variance and trends in the time series. ARIMA with autonomous parameter selection takes this a step further by automatically optimizing the model’s parameters (p, d, q) using techniques like grid search or other statistical criteria, ensuring that the model is well-suited to the data’s underlying structure for more accurate predictions. To address this, we incorporated both an ARIMA component and an AUTO-ARIMA component, enabling the prediction of future time series data points for each sensor.
+
+<br>
+
+Working on the RTDIP Project within AMOS has been a fantastic journey, highlighting the importance of people and teamwork in agile development. By focusing on enhancing data quality, we’ve significantly boosted the reliability, consistency, and usability of the data going through the RTDIP pipeline.
+
+To look back, our regular team meetings were the key to our success. Through open communication and collaboration, we tackled challenges and kept improving our processes. This showed us the power of working together in an agile framework and growing as a dedicated SCRUM team.
+
+We’re excited about the future and how these advancements will help data scientists and engineers make better decisions.
+
+<br>
+
+<small>1 Designed by Freepik</small><br>
+<small>2 Michael Stonebraker, Ihab F. Ilyas: Data Integration: The Current Status and the Way Forward. IEEE Data Eng. Bull. 41(2) (2018)</small>
diff --git a/...ence/pipelines/data_quality/data_manipulation/spark/dimensionality_reduction.md b/...ence/pipelines/data_quality/data_manipulation/spark/dimensionality_reduction.md
@@ -0,0 +1 @@
+::: src.sdk.python.rtdip_sdk.pipelines.data_quality.data_manipulation.spark.dimensionality_reduction
diff --git a/...reference/pipelines/data_quality/data_manipulation/spark/duplicate_detection.md b/...reference/pipelines/data_quality/data_manipulation/spark/duplicate_detection.md
@@ -0,0 +1 @@
+::: src.sdk.python.rtdip_sdk.pipelines.data_quality.data_manipulation.spark.duplicate_detection
diff --git a/...ode-reference/pipelines/data_quality/data_manipulation/spark/flatline_filter.md b/...ode-reference/pipelines/data_quality/data_manipulation/spark/flatline_filter.md
@@ -0,0 +1 @@
+::: src.sdk.python.rtdip_sdk.pipelines.data_quality.data_manipulation.spark.flatline_filter
diff --git a/...-reference/pipelines/data_quality/data_manipulation/spark/gaussian_smoothing.md b/...-reference/pipelines/data_quality/data_manipulation/spark/gaussian_smoothing.md
@@ -0,0 +1 @@
+::: src.sdk.python.rtdip_sdk.pipelines.data_quality.data_manipulation.spark.gaussian_smoothing
diff --git a/...-reference/pipelines/data_quality/data_manipulation/spark/interval_filtering.md b/...-reference/pipelines/data_quality/data_manipulation/spark/interval_filtering.md
@@ -0,0 +1 @@
+::: src.sdk.python.rtdip_sdk.pipelines.data_quality.data_manipulation.spark.interval_filtering
diff --git a/...nce/pipelines/data_quality/data_manipulation/spark/k_sigma_anomaly_detection.md b/...nce/pipelines/data_quality/data_manipulation/spark/k_sigma_anomaly_detection.md
@@ -0,0 +1 @@
+::: src.sdk.python.rtdip_sdk.pipelines.data_quality.data_manipulation.spark.k_sigma_anomaly_detection
diff --git a/...ence/pipelines/data_quality/data_manipulation/spark/missing_value_imputation.md b/...ence/pipelines/data_quality/data_manipulation/spark/missing_value_imputation.md
@@ -0,0 +1,2 @@
+::: src.sdk.python.rtdip_sdk.pipelines.data_quality.data_manipulation.spark.missing_value_imputation
+
diff --git a/...pipelines/data_quality/data_manipulation/spark/normalization/denormalization.md b/...pipelines/data_quality/data_manipulation/spark/normalization/denormalization.md
@@ -0,0 +1 @@
+::: src.sdk.python.rtdip_sdk.pipelines.data_quality.data_manipulation.spark.normalization.denormalization
diff --git a/...e/pipelines/data_quality/data_manipulation/spark/normalization/normalization.md b/...e/pipelines/data_quality/data_manipulation/spark/normalization/normalization.md
@@ -0,0 +1 @@
+::: src.sdk.python.rtdip_sdk.pipelines.data_quality.data_manipulation.spark.normalization.normalization
diff --git a/...elines/data_quality/data_manipulation/spark/normalization/normalization_mean.md b/...elines/data_quality/data_manipulation/spark/normalization/normalization_mean.md
@@ -0,0 +1 @@
+::: src.sdk.python.rtdip_sdk.pipelines.data_quality.data_manipulation.spark.normalization.normalization_mean
diff --git a/...ines/data_quality/data_manipulation/spark/normalization/normalization_minmax.md b/...ines/data_quality/data_manipulation/spark/normalization/normalization_minmax.md
@@ -0,0 +1 @@
+::: src.sdk.python.rtdip_sdk.pipelines.data_quality.data_manipulation.spark.normalization.normalization_minmax
diff --git a/...ines/data_quality/data_manipulation/spark/normalization/normalization_zscore.md b/...ines/data_quality/data_manipulation/spark/normalization/normalization_zscore.md
@@ -0,0 +1 @@
+::: src.sdk.python.rtdip_sdk.pipelines.data_quality.data_manipulation.spark.normalization.normalization_zscore
diff --git a/...nce/pipelines/data_quality/data_manipulation/spark/out_of_range_value_filter.md b/...nce/pipelines/data_quality/data_manipulation/spark/out_of_range_value_filter.md
@@ -0,0 +1 @@
+::: src.sdk.python.rtdip_sdk.pipelines.data_quality.data_manipulation.spark.out_of_range_value_filter
diff --git a/...dk/code-reference/pipelines/data_quality/monitoring/spark/check_value_ranges.md b/...dk/code-reference/pipelines/data_quality/monitoring/spark/check_value_ranges.md
@@ -0,0 +1 @@
+::: src.sdk.python.rtdip_sdk.pipelines.data_quality.monitoring.spark.check_value_ranges
diff --git a/...dk/code-reference/pipelines/data_quality/monitoring/spark/flatline_detection.md b/...dk/code-reference/pipelines/data_quality/monitoring/spark/flatline_detection.md
@@ -0,0 +1 @@
+::: src.sdk.python.rtdip_sdk.pipelines.data_quality.monitoring.spark.flatline_detection
diff --git a/.../spark/data_quality/great_expectations.md → ...ty/monitoring/spark/great_expectations.md b/.../spark/data_quality/great_expectations.md → ...ty/monitoring/spark/great_expectations.md
@@ -2,4 +2,4 @@
 
 Great Expectations is a Python-based open-source library for validating, documenting, and profiling your data. It helps you to maintain data quality and improve communication about data between teams.
 
-::: src.sdk.python.rtdip_sdk.pipelines.monitoring.spark.data_quality.great_expectations_data_quality
+::: src.sdk.python.rtdip_sdk.pipelines.data_quality.monitoring.spark.great_expectations_data_quality
diff --git a/...rence/pipelines/data_quality/monitoring/spark/identify_missing_data_interval.md b/...rence/pipelines/data_quality/monitoring/spark/identify_missing_data_interval.md
@@ -0,0 +1 @@
+::: src.sdk.python.rtdip_sdk.pipelines.data_quality.monitoring.spark.identify_missing_data_interval
diff --git a/...erence/pipelines/data_quality/monitoring/spark/identify_missing_data_pattern.md b/...erence/pipelines/data_quality/monitoring/spark/identify_missing_data_pattern.md
@@ -0,0 +1 @@
+::: src.sdk.python.rtdip_sdk.pipelines.data_quality.monitoring.spark.identify_missing_data_pattern
diff --git a/docs/sdk/code-reference/pipelines/data_quality/monitoring/spark/moving_average.md b/docs/sdk/code-reference/pipelines/data_quality/monitoring/spark/moving_average.md
@@ -0,0 +1 @@
+::: src.sdk.python.rtdip_sdk.pipelines.data_quality.monitoring.spark.moving_average
diff --git a/docs/sdk/code-reference/pipelines/forecasting/spark/arima.md b/docs/sdk/code-reference/pipelines/forecasting/spark/arima.md
@@ -0,0 +1 @@
+::: src.sdk.python.rtdip_sdk.pipelines.forecasting.spark.arima
diff --git a/docs/sdk/code-reference/pipelines/forecasting/spark/auto_arima.md b/docs/sdk/code-reference/pipelines/forecasting/spark/auto_arima.md
@@ -0,0 +1 @@
+::: src.sdk.python.rtdip_sdk.pipelines.forecasting.spark.auto_arima
diff --git a/docs/sdk/code-reference/pipelines/forecasting/spark/data_binning.md b/docs/sdk/code-reference/pipelines/forecasting/spark/data_binning.md
@@ -0,0 +1 @@
+::: src.sdk.python.rtdip_sdk.pipelines.forecasting.spark.data_binning
diff --git a/docs/sdk/code-reference/pipelines/forecasting/spark/k_nearest_neighbors.md b/docs/sdk/code-reference/pipelines/forecasting/spark/k_nearest_neighbors.md
@@ -0,0 +1 @@
+::: src.sdk.python.rtdip_sdk.pipelines.forecasting.spark.k_nearest_neighbors
diff --git a/docs/sdk/code-reference/pipelines/forecasting/spark/linear_regression.md b/docs/sdk/code-reference/pipelines/forecasting/spark/linear_regression.md
@@ -0,0 +1 @@
+::: src.sdk.python.rtdip_sdk.pipelines.forecasting.spark.linear_regression
diff --git a/environment.yml b/environment.yml
@@ -68,6 +68,8 @@ dependencies:
     - black>=24.1.0
     - joblib==1.3.2,<2.0.0
     - great-expectations>=0.18.8,<1.0.0
+    - statsmodels>=0.14.1,<0.15.0
+    - pmdarima>=2.0.4
     - pip:
           - databricks-sdk>=0.20.0,<1.0.0
           - dependency-injector>=4.41.0,<5.0.0

diff --git a/mkdocs.yml b/mkdocs.yml
@@ -235,10 +235,38 @@ nav:
                   - Azure Key Vault: sdk/code-reference/pipelines/secrets/azure_key_vault.md
               - Deploy:
                   - Databricks: sdk/code-reference/pipelines/deploy/databricks.md
-              - Monitoring:
-                  - Data Quality: 
-                      - Great Expectations: 
-                        - Data Quality Monitoring: sdk/code-reference/pipelines/monitoring/spark/data_quality/great_expectations.md
+              - Data Quality:
+                  - Monitoring:
+                      - Check Value Ranges: sdk/code-reference/pipelines/data_quality/monitoring/spark/check_value_ranges.md
+                      - Great Expectations:
+                          - Data Quality Monitoring: sdk/code-reference/pipelines/data_quality/monitoring/spark/great_expectations.md
+                      - Flatline Detection: sdk/code-reference/pipelines/data_quality/monitoring/spark/flatline_detection.md
+                      - Identify Missing Data:
+                        - Interval Based: sdk/code-reference/pipelines/data_quality/monitoring/spark/identify_missing_data_interval.md
+                        - Pattern Based: sdk/code-reference/pipelines/data_quality/monitoring/spark/identify_missing_data_pattern.md
+                      - Moving Average: sdk/code-reference/pipelines/data_quality/monitoring/spark/moving_average.md
+                  - Data Manipulation: 
+                      - Duplicate Detetection: sdk/code-reference/pipelines/data_quality/data_manipulation/spark/duplicate_detection.md
+                      - Out of Range Value Filter: sdk/code-reference/pipelines/data_quality/data_manipulation/spark/out_of_range_value_filter.md
+                      - Flatline Filter: sdk/code-reference/pipelines/data_quality/data_manipulation/spark/flatline_filter.md
+                      - Gaussian Smoothing: sdk/code-reference/pipelines/data_quality/data_manipulation/spark/gaussian_smoothing.md
+                      - Dimensionality Reduction: sdk/code-reference/pipelines/data_quality/data_manipulation/spark/dimensionality_reduction.md
+                      - Interval Filtering: sdk/code-reference/pipelines/data_quality/data_manipulation/spark/interval_filtering.md
+                      - K-Sigma Anomaly Detection: sdk/code-reference/pipelines/data_quality/data_manipulation/spark/k_sigma_anomaly_detection.md
+                      - Missing Value Imputation: sdk/code-reference/pipelines/data_quality/data_manipulation/spark/missing_value_imputation.md
+                      - Normalization:
+                          - Normalization: sdk/code-reference/pipelines/data_quality/data_manipulation/spark/normalization/normalization.md
+                          - Normalization Mean: sdk/code-reference/pipelines/data_quality/data_manipulation/spark/normalization/normalization_mean.md
+                          - Normalization MinMax: sdk/code-reference/pipelines/data_quality/data_manipulation/spark/normalization/normalization_minmax.md
+                          - Normalization ZScore: sdk/code-reference/pipelines/data_quality/data_manipulation/spark/normalization/normalization_zscore.md
+                          - Denormalization: sdk/code-reference/pipelines/data_quality/data_manipulation/spark/normalization/denormalization.md
+              - Forecasting:
+                  - Data Binning: sdk/code-reference/pipelines/forecasting/spark/data_binning.md
+                  - Linear Regression: sdk/code-reference/pipelines/forecasting/spark/linear_regression.md
+                  - Arima: sdk/code-reference/pipelines/forecasting/spark/arima.md
+                  - Auto Arima: sdk/code-reference/pipelines/forecasting/spark/auto_arima.md
+                  - K Nearest Neighbors: sdk/code-reference/pipelines/forecasting/spark/k_nearest_neighbors.md
+
           - Jobs: sdk/pipelines/jobs.md
           - Deploy:
               - Databricks Workflows: sdk/pipelines/deploy/databricks.md
@@ -330,4 +358,4 @@ nav:
       - blog/index.md
   - University:
       - University: university/overview.md
-
+
diff --git a/setup.py b/setup.py
@@ -1,4 +1,4 @@
-# Copyright 2022 RTDIP
+# Copyright 2025 RTDIP
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -46,6 +46,8 @@
     "langchain>=0.2.0,<0.3.0",
     "langchain-community>=0.2.0,<0.3.0",
     "openai>=1.13.3,<2.0.0",
+    "statsmodels>=0.14.1,<0.15.0",
+    "pmdarima>=2.0.4",
 ]
 
 PYSPARK_PACKAGES = [
@@ -71,6 +73,7 @@
     "joblib>=1.3.2,<2.0.0",
     "sqlparams>=5.1.0,<6.0.0",
     "entsoe-py>=0.5.10,<1.0.0",
+    "numpy>=1.23.4,<2.0.0",
 ]
 
 EXTRAS_DEPENDENCIES: dict[str, list[str]] = {

diff --git a/src/api/requirements.txt b/src/api/requirements.txt
@@ -1,6 +1,6 @@
 # Do not include azure-functions-worker as it may conflict with the Azure Functions platform
 azure-functions==1.18.0
-fastapi==0.110.0
+fastapi==0.115.6
 pydantic==2.6.0
 # turbodbc==4.11.0
 pyodbc==4.0.39
@@ -10,7 +10,7 @@ azure-identity==1.17.0
 oauthlib>=3.2.2
 pandas>=2.0.1,<3.0.0
 numpy==1.26.4
-jinja2==3.1.4
+jinja2==3.1.5
 pytz==2024.1
 semver==3.0.2
 xlrd==2.0.1
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		::: src.sdk.python.rtdip_sdk.pipelines.data_quality.data_manipulation.spark.dimensionality_reduction
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		::: src.sdk.python.rtdip_sdk.pipelines.data_quality.data_manipulation.spark.missing_value_imputation
Original file line number	Diff line number	Diff line change
Expand Up		@@ -2,4 +2,4 @@

		Great Expectations is a Python-based open-source library for validating, documenting, and profiling your data. It helps you to maintain data quality and improve communication about data between teams.

		::: src.sdk.python.rtdip_sdk.pipelines.monitoring.spark.data_quality.great_expectations_data_quality
		::: src.sdk.python.rtdip_sdk.pipelines.data_quality.monitoring.spark.great_expectations_data_quality
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		::: src.sdk.python.rtdip_sdk.pipelines.forecasting.spark.arima