From 5a8b2f2a6934667487d6411dff9cb98b4dd39bf2 Mon Sep 17 00:00:00 2001 From: Jeroen Date: Thu, 19 Jun 2025 15:43:32 +0200 Subject: [PATCH] Adding Docker setup --- .dockerignore | 15 +++ Dockerfile | 18 +++ README (files-docker-compose).md | 67 +++++++++++ docker-compose.prod.yml | 111 ++++++++++++++++++ docker-compose.yml | 33 ++++++ plugins/files/files-docker--meltano.lock | 11 ++ .../files/files-docker-compose--meltano.lock | 11 ++ 7 files changed, 266 insertions(+) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 README (files-docker-compose).md create mode 100644 docker-compose.prod.yml create mode 100644 docker-compose.yml create mode 100644 plugins/files/files-docker--meltano.lock create mode 100644 plugins/files/files-docker-compose--meltano.lock diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..5559c57 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,15 @@ +/.git + +# .gitignore +/venv +/.meltano +/.env +/ui.cfg +/output + +# transform/.gitignore +/transform/target/ +/transform/dbt_modules/ +/transform/logs/ + +# custom \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ac6f734 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +# registry.gitlab.com/meltano/meltano:latest is also available in GitLab Registry +ARG MELTANO_IMAGE=meltano/meltano:latest +FROM $MELTANO_IMAGE + +WORKDIR /project + +# Install any additional requirements +COPY ./requirements.txt . +RUN pip install -r requirements.txt + +# Copy over Meltano project directory +COPY . . +RUN meltano install + +# Don't allow changes to containerized project files +ENV MELTANO_PROJECT_READONLY=1 + +ENTRYPOINT ["meltano"] diff --git a/README (files-docker-compose).md b/README (files-docker-compose).md new file mode 100644 index 0000000..35441d0 --- /dev/null +++ b/README (files-docker-compose).md @@ -0,0 +1,67 @@ +# Meltano & Docker Compose + +*[This file](https://gitlab.com/meltano/files-docker-compose/-/blob/master/bundle/README.md) has been added to your project for convenience and reference only. Feel free to delete it.* + +## Getting started + +1. Start the services in the background: + + ```bash + docker compose up -d + ``` + +### Helpful commands + +- `docker compose run meltano {subcommand}`: Run a [`meltano` CLI command](https://meltano.com/docs/command-line-interface.html) inside your container. +- `docker compose logs`: See all logs. +- `docker compose logs {service}`: See logs for a particular service, e.g. `meltano`. + +## Optional services + +If these services are not relevant to you, feel free to delete their commented sections. + +### Airflow + +If you are using the [Airflow orchestrator](https://meltano.com/docs/orchestration.html) and would like to run it using Docker Compose, follow these steps: + +1. Uncomment the `airflow-webserver` and `airflow-scheduler` services. +1. Start the new services: + + ```bash + docker compose up -d + ``` + +1. Open the Airflow web interface at . + +## Production usage + +A `docker-compose.prod.yml` file is included that represents a [production-grade](https://meltano.com/docs/production.html) setup of a [containerized Meltano project](https://meltano.com/docs/containerization.html). + +If this is not relevant to you, feel free to delete it. + +### Dependencies + +The production configuration depends on a `Dockerfile` being present in your project. + +If you haven't already, add the appropriate `Dockerfile` and `.dockerignore` files to your project by adding the [`docker` file bundle](https://gitlab.com/meltano/files-docker): + +```bash +meltano add files docker +``` + +### Usage + +Please ensure you do the following before deploying to production: + +1. If you are using the [Airflow orchestrator](#airflow) and would like to run it using Docker Compose, uncomment the Airflow services, network, and volume, and add `psycopg2` to `airflow`'s `pip_url` in `meltano.yml` as described in the ["Deployment in Production" guide](https://meltano.com/docs/production.html#airflow-orchestrator). If not, feel free to delete the commented sections. +1. Change the database password for `meltano-system-db` (and `airflow-metadata-db`): look for `# CHANGE ME`. +1. Update the database connection URIs under `x-meltano-env` (and `x-airflow-env`) to reflect the changed passwords. +1. Add any environment variables from `.env` and your local environment that are needed for production under `x-meltano-env`. +1. Change the image name and tag under `x-meltano-image` to something that makes sense for your project. +1. Start the services in the background: + + ```bash + docker compose -f docker-compose.prod.yml up -d + ``` + +If you've made changes to your project and need to rebuild your project-specific image, run `docker compose -f docker-compose.prod.yml up -d --build`. diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml new file mode 100644 index 0000000..517e756 --- /dev/null +++ b/docker-compose.prod.yml @@ -0,0 +1,111 @@ +x-meltano-image: &meltano-image + image: meltano-demo-project:dev # Change me to a name and tag that makes sense for your project + build: . + +x-meltano-env: &meltano-env + MELTANO_DATABASE_URI: postgresql://postgres:postgres@meltano-system-db/meltano + # Add any additional Meltano configuration environment variables here + +# # Uncomment if you are using the Airflow orchestrator, delete otherwise +# x-airflow-env: &airflow-env +# AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgres://postgres:postgres@airflow-metadata-db/airflow +# AIRFLOW__CORE__EXECUTOR: LocalExecutor + +services: + meltano: + <<: *meltano-image + command: dragon + environment: + <<: *meltano-env + # # Uncomment if you are using the Airflow orchestrator, delete otherwise + # <<: *airflow-env + volumes: + - meltano_elt_logs_data:/project/.meltano/logs/elt + expose: + - 5000 + ports: + - 5000:5000 + depends_on: + - meltano-system-db + networks: + - meltano + restart: unless-stopped + + meltano-system-db: + image: postgres + environment: + POSTGRES_PASSWORD: postgres # CHANGE ME + POSTGRES_DB: meltano + PGDATA: /var/lib/postgresql/data/pgdata + volumes: + - meltano_postgresql_data:/var/lib/postgresql/data + expose: + - 5432 + networks: + - meltano + restart: unless-stopped + + # # Uncomment if you are using the Airflow orchestrator, delete otherwise + # airflow-scheduler: + # <<: *meltano-image + # command: invoke airflow scheduler + # environment: + # <<: *meltano-env + # <<: *airflow-env + # volumes: + # - meltano_elt_logs_data:/project/.meltano/logs/elt + # expose: + # - 8793 + # depends_on: + # - meltano-system-db + # - airflow-metadata-db + # networks: + # - meltano + # - airflow + # restart: unless-stopped + # + # airflow-webserver: + # <<: *meltano-image + # command: invoke airflow webserver + # environment: + # <<: *meltano-env + # <<: *airflow-env + # expose: + # - 8080 + # ports: + # - 8080:8080 + # depends_on: + # - meltano-system-db + # - airflow-metadata-db + # networks: + # - meltano + # - airflow + # restart: unless-stopped + # + # airflow-metadata-db: + # image: postgres + # environment: + # POSTGRES_PASSWORD: postgres # CHANGE ME + # POSTGRES_DB: airflow + # PGDATA: /var/lib/postgresql/data/pgdata + # volumes: + # - airflow_postgresql_data:/var/lib/postgresql/data + # expose: + # - 5432 + # networks: + # - airflow + # restart: unless-stopped + +networks: + meltano: + # # Uncomment if you are using the Airflow orchestrator, delete otherwise + # airflow: + +volumes: + meltano_postgresql_data: + driver: local + meltano_elt_logs_data: + driver: local + # # Uncomment if you are using the Airflow orchestrator, delete otherwise + # airflow_postgresql_data: + # driver: local diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..fecc1ed --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,33 @@ +x-meltano-image: &meltano-image + image: lakehouse/sarens-integration:latest + +services: + meltano: + <<: *meltano-image + command: dragon + restart: unless-stopped + networks: + - db_network + + # # Uncomment if you are using the Airflow orchestrator, delete otherwise + # airflow-scheduler: + # <<: *meltano-image + # command: invoke airflow scheduler + # expose: + # - 8793 + # restart: unless-stopped + # + # airflow-webserver: + # <<: *meltano-image + # command: invoke airflow webserver + # expose: + # - 8080 + # ports: + # - 8080:8080 + # restart: unless-stopped + + +networks: + db_network: + external: true + name: db_network \ No newline at end of file diff --git a/plugins/files/files-docker--meltano.lock b/plugins/files/files-docker--meltano.lock new file mode 100644 index 0000000..5f9275f --- /dev/null +++ b/plugins/files/files-docker--meltano.lock @@ -0,0 +1,11 @@ +{ + "plugin_type": "files", + "name": "files-docker", + "namespace": "files_docker", + "variant": "meltano", + "label": "Docker", + "docs": "https://hub.meltano.com/files/files-docker--meltano", + "repo": "https://github.com/meltano/files-docker", + "pip_url": "git+https://github.com/meltano/files-docker.git", + "logo_url": "https://hub.meltano.com/assets/logos/files/docker.png" +} diff --git a/plugins/files/files-docker-compose--meltano.lock b/plugins/files/files-docker-compose--meltano.lock new file mode 100644 index 0000000..620323f --- /dev/null +++ b/plugins/files/files-docker-compose--meltano.lock @@ -0,0 +1,11 @@ +{ + "plugin_type": "files", + "name": "files-docker-compose", + "namespace": "files_docker_compose", + "variant": "meltano", + "label": "Docker Compose", + "docs": "https://hub.meltano.com/files/files-docker-compose--meltano", + "repo": "https://github.com/meltano/files-docker-compose", + "pip_url": "git+https://github.com/meltano/files-docker-compose.git", + "logo_url": "https://hub.meltano.com/assets/logos/files/docker-compose.png" +}