diff --git a/README.md b/README.md index a8fc942..70186d2 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,7 @@ client = Client(host="your-glassflow-etl-url") ```python pipeline_config = { + "version": "v2", "pipeline_id": "my-pipeline-id", "source": { "type": "kafka", @@ -63,31 +64,6 @@ pipeline_config = { { "consumer_group_initial_offset": "latest", "name": "users", - "schema": { - "type": "json", - "fields": [ - { - "name": "event_id", - "type": "string" - }, - { - "name": "user_id", - "type": "string" - }, - { - "name": "name", - "type": "string" - }, - { - "name": "email", - "type": "string" - }, - { - "name": "created_at", - "type": "string" - } - ] - }, "deduplication": { "enabled": True, "id_field": "event_id", @@ -110,11 +86,14 @@ pipeline_config = { "secure": False, "max_batch_size": 1000, "max_delay_time": "30s", - "table": "users_dedup", - "table_mapping": [ + "table": "users_dedup" + }, + "schema": { + "fields": [ { "source_id": "users", - "field_name": "event_id", + "name": "event_id", + "type": "string", "column_name": "event_id", "column_type": "UUID" }, @@ -126,19 +105,22 @@ pipeline_config = { }, { "source_id": "users", - "field_name": "created_at", + "name": "created_at", + "type": "string", "column_name": "created_at", "column_type": "DateTime" }, { "source_id": "users", - "field_name": "name", + "name": "name", + "type": "string", "column_name": "name", "column_type": "String" }, { "source_id": "users", - "field_name": "email", + "name": "email", + "type": "string", "column_name": "email", "column_type": "String" } diff --git a/VERSION b/VERSION index 40c341b..7c69a55 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.6.0 +3.7.0 diff --git a/src/glassflow/etl/models/pipeline.py b/src/glassflow/etl/models/pipeline.py index 9f31b0a..629e29a 100644 --- a/src/glassflow/etl/models/pipeline.py +++ b/src/glassflow/etl/models/pipeline.py @@ -25,7 +25,7 @@ class PipelineStatus(CaseInsensitiveStrEnum): class PipelineConfig(BaseModel): - version: str = Field(default="2.0.0") + version: str = Field(default="v2") pipeline_id: str name: Optional[str] = Field(default=None) source: SourceConfig