diff --git a/README.md b/README.md index 7251b6a..304b3ad 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,6 @@ meltano run tap-spreadsheets-anywhere target-postgres -meltano invoke dbt-postgres:run \ No newline at end of file +meltano invoke dbt-postgres:run + + +meltano --environment=sarens-demo run tap-spreadsheets-anywhere target-postgres dbt-postgres:run diff --git a/docker-compose.yml b/docker-compose.yml index c5ff720..2016d34 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,10 +4,12 @@ x-meltano-image: &meltano-image services: meltano: <<: *meltano-image - command: dragon + command: "--environment=sarens-demo run tap-spreadsheets-anywhere target-postgres dbt-postgres:run" restart: unless-stopped networks: - db_network + volumes: + - /home/sarens-data:/sarens-data # # Uncomment if you are using the Airflow orchestrator, delete otherwise # airflow-scheduler: diff --git a/meltano.yml b/meltano.yml index 182a75d..ed08516 100644 --- a/meltano.yml +++ b/meltano.yml @@ -75,9 +75,6 @@ environments: worksheet_name: Tabel sample_rate: 1 max_sampling_read: 25000 - select: - - departments.* - - '!departments._*' loaders: - name: target-postgres config: @@ -94,7 +91,95 @@ environments: user: postgres port: 5432 schema: clean -- name: staging +- name: sarens-demo + config: + plugins: + extractors: + - name: tap-spreadsheets-anywhere + config: + tables: + - path: file:///sarens-data/ + name: departments + pattern: Applicable Organizations.xlsx + start_date: '2000-01-01T00:00:00Z' + key_properties: [] + format: excel + worksheet_name: AO + - path: file:///sarens-data/ + name: positions + pattern: "HR002.*" + start_date: '2000-01-01T00:00:00Z' + key_properties: [] + format: excel + worksheet_name: HR002 - Positions Report (INTER + skip_initial: 8 + sample_rate: 1 + max_sampling_read: 1000 + - path: file:///sarens-data/ + name: workers + pattern: "HR006.*" + start_date: '2000-01-01T00:00:00Z' + key_properties: [] + format: excel + worksheet_name: HR006 - All Users Report (Activ + skip_initial: 8 + sample_rate: 1 + max_sampling_read: 25000 + - path: file:///sarens-data/ + name: performance_review_steps + pattern: "PER001.*" + start_date: '2000-01-01T00:00:00Z' + key_properties: [] + format: excel + worksheet_name: PER001 - Performance Review (2) + skip_initial: 8 + sample_rate: 1 + max_sampling_read: 25000 + - path: file:///sarens-data/ + name: performance_review_total_scoring + pattern: "PER002.*" + start_date: '2000-01-01T00:00:00Z' + key_properties: [] + format: excel + worksheet_name: PER002 - Performance Review (2) + skip_initial: 8 + sample_rate: 1 + max_sampling_read: 25000 + - path: file:///sarens-data/ + name: performance_review_sub_scoring + pattern: "PER003.*" + start_date: '2000-01-01T00:00:00Z' + key_properties: [] + format: excel + worksheet_name: PER003 - Performance Review (2) + skip_initial: 8 + sample_rate: 1 + max_sampling_read: 25000 + - path: file:///sarens-data/ + name: absenteisme + pattern: "Absenteisme.*" + start_date: '2000-01-01T00:00:00Z' + key_properties: [] + format: excel + worksheet_name: Tabel + sample_rate: 1 + max_sampling_read: 25000 + loaders: + - name: target-postgres + config: + database: lakehouse_sarens + host: lakehouse-sarens-db-1 + load_method: append-only + user: lakehouse-sarens + default_target_schema: raw + utilities: + - name: dbt-postgres + config: + host: lakehouse-sarens-db-1 + dbname: lakehouse-sarens + user: lakehouse-sarens + port: 5432 + schema: clean - name: prod plugins: extractors: