Source
yaml
id: fivetran-sync-parallel-dbt-core
namespace: company.team
tasks:
- id: data_ingestion
type: io.kestra.plugin.core.flow.Parallel
tasks:
- id: salesforce
type: io.kestra.plugin.fivetran.connectors.Sync
connectorId: vesicle_movement
- id: stripe
type: io.kestra.plugin.fivetran.connectors.Sync
connectorId: cell_delivery
- id: google_analytics
type: io.kestra.plugin.fivetran.connectors.Sync
connectorId: equivocal_sandy
- id: facebook_ads
type: io.kestra.plugin.fivetran.connectors.Sync
connectorId: molecule_transport
- id: dbt_core
type: io.kestra.plugin.core.flow.WorkingDirectory
tasks:
- id: clone_repository
type: io.kestra.plugin.git.Clone
url: https://github.com/kestra-io/dbt-demo
branch: main
- id: dbt_build
type: io.kestra.plugin.dbt.cli.Build
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
containerImage: ghcr.io/kestra-io/dbt-bigquery:latest
dbtPath: /usr/local/bin/dbt
inputFiles:
.profile/profiles.yml: |
jaffle_shop:
outputs:
dev:
type: bigquery
dataset: your_big_query_dataset_name
project: your_big_query_project
fixed_retries: 1
keyfile: sa.json
location: EU
method: service-account
priority: interactive
threads: 8
timeout_seconds: 300
target: dev
sa.json: "{{ secret('GCP_CREDS') }}"
pluginDefaults:
- type: io.kestra.plugin.fivetran.connectors.Sync
values:
apiKey: "{{ secret('FIVETRAN_API_KEY') }}"
apiSecret: "{{ secret('FIVETRAN_API_SECRET') }}"
About this blueprint
dbt Git Kestra
This flow runs Fivetran syncs in parallel and then runs dbt core's CLI commands.
The flow assumes that you store the Fivetran API credentials, referenced in
the pluginDefaults, as secrets.
More Related Blueprints