Adding Docker setup

.dockerignore

@@ -0,0 +1,15 @@
+/.git
+
+# .gitignore
+/venv
+/.meltano
+/.env
+/ui.cfg
+/output
+
+# transform/.gitignore
+/transform/target/
+/transform/dbt_modules/
+/transform/logs/
+
+# custom

Dockerfile

@@ -0,0 +1,18 @@
+# registry.gitlab.com/meltano/meltano:latest is also available in GitLab Registry
+ARG MELTANO_IMAGE=meltano/meltano:latest
+FROM $MELTANO_IMAGE
+
+WORKDIR /project
+
+# Install any additional requirements
+COPY ./requirements.txt .
+RUN pip install -r requirements.txt
+
+# Copy over Meltano project directory
+COPY . .
+RUN meltano install
+
+# Don't allow changes to containerized project files
+ENV MELTANO_PROJECT_READONLY=1
+
+ENTRYPOINT ["meltano"]

README (files-docker-compose).md

@@ -0,0 +1,67 @@
+# Meltano & Docker Compose
+
+*[This file](https://gitlab.com/meltano/files-docker-compose/-/blob/master/bundle/README.md) has been added to your project for convenience and reference only. Feel free to delete it.*
+
+## Getting started
+
+1. Start the services in the background:
+
+    ```bash
+    docker compose up -d
+    ```
+
+### Helpful commands
+
+- `docker compose run meltano {subcommand}`: Run a [`meltano` CLI command](https://meltano.com/docs/command-line-interface.html) inside your container.
+- `docker compose logs`: See all logs.
+- `docker compose logs {service}`: See logs for a particular service, e.g. `meltano`.
+
+## Optional services
+
+If these services are not relevant to you, feel free to delete their commented sections.
+
+### Airflow
+
+If you are using the [Airflow orchestrator](https://meltano.com/docs/orchestration.html) and would like to run it using Docker Compose, follow these steps:
+
+1. Uncomment the `airflow-webserver` and `airflow-scheduler` services.
+1. Start the new services:
+
+    ```bash
+    docker compose up -d
+    ```
+
+1. Open the Airflow web interface at <http://localhost:8080>.
+
+## Production usage
+
+A `docker-compose.prod.yml` file is included that represents a [production-grade](https://meltano.com/docs/production.html) setup of a [containerized Meltano project](https://meltano.com/docs/containerization.html).
+
+If this is not relevant to you, feel free to delete it.
+
+### Dependencies
+
+The production configuration depends on a `Dockerfile` being present in your project.
+
+If you haven't already, add the appropriate `Dockerfile` and `.dockerignore` files to your project by adding the [`docker` file bundle](https://gitlab.com/meltano/files-docker):
+
+```bash
+meltano add files docker
+```
+
+### Usage
+
+Please ensure you do the following before deploying to production:
+
+1. If you are using the [Airflow orchestrator](#airflow) and would like to run it using Docker Compose, uncomment the Airflow services, network, and volume, and add `psycopg2` to `airflow`'s `pip_url` in `meltano.yml` as described in the ["Deployment in Production" guide](https://meltano.com/docs/production.html#airflow-orchestrator). If not, feel free to delete the commented sections.
+1. Change the database password for `meltano-system-db` (and `airflow-metadata-db`): look for `# CHANGE ME`.
+1. Update the database connection URIs under `x-meltano-env` (and `x-airflow-env`) to reflect the changed passwords.
+1. Add any environment variables from `.env` and your local environment that are needed for production under `x-meltano-env`.
+1. Change the image name and tag under `x-meltano-image` to something that makes sense for your project.
+1. Start the services in the background:
+
+    ```bash
+    docker compose -f docker-compose.prod.yml up -d
+    ```
+
+If you've made changes to your project and need to rebuild your project-specific image, run `docker compose -f docker-compose.prod.yml up -d --build`.

docker-compose.prod.yml

@@ -0,0 +1,111 @@
+x-meltano-image: &meltano-image
+  image: meltano-demo-project:dev # Change me to a name and tag that makes sense for your project
+  build: .
+
+x-meltano-env: &meltano-env
+  MELTANO_DATABASE_URI: postgresql://postgres:postgres@meltano-system-db/meltano
+  # Add any additional Meltano configuration environment variables here
+
+# # Uncomment if you are using the Airflow orchestrator, delete otherwise
+# x-airflow-env: &airflow-env
+#   AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgres://postgres:postgres@airflow-metadata-db/airflow
+#   AIRFLOW__CORE__EXECUTOR: LocalExecutor
+
+services:
+  meltano:
+    <<: *meltano-image
+    command: dragon
+    environment:
+      <<: *meltano-env
+      # # Uncomment if you are using the Airflow orchestrator, delete otherwise
+      # <<: *airflow-env
+    volumes:
+      - meltano_elt_logs_data:/project/.meltano/logs/elt
+    expose:
+      - 5000
+    ports:
+      - 5000:5000
+    depends_on:
+      - meltano-system-db
+    networks:
+      - meltano
+    restart: unless-stopped
+
+  meltano-system-db:
+    image: postgres
+    environment:
+      POSTGRES_PASSWORD: postgres # CHANGE ME
+      POSTGRES_DB: meltano
+      PGDATA: /var/lib/postgresql/data/pgdata
+    volumes:
+      - meltano_postgresql_data:/var/lib/postgresql/data
+    expose:
+      - 5432
+    networks:
+      - meltano
+    restart: unless-stopped
+
+  # # Uncomment if you are using the Airflow orchestrator, delete otherwise
+  # airflow-scheduler:
+  #   <<: *meltano-image
+  #   command: invoke airflow scheduler
+  #   environment:
+  #     <<: *meltano-env
+  #     <<: *airflow-env
+  #   volumes:
+  #     - meltano_elt_logs_data:/project/.meltano/logs/elt
+  #   expose:
+  #     - 8793
+  #   depends_on:
+  #     - meltano-system-db
+  #     - airflow-metadata-db
+  #   networks:
+  #     - meltano
+  #     - airflow
+  #   restart: unless-stopped
+  #
+  # airflow-webserver:
+  #   <<: *meltano-image
+  #   command: invoke airflow webserver
+  #   environment:
+  #     <<: *meltano-env
+  #     <<: *airflow-env
+  #   expose:
+  #     - 8080
+  #   ports:
+  #     - 8080:8080
+  #   depends_on:
+  #     - meltano-system-db
+  #     - airflow-metadata-db
+  #   networks:
+  #     - meltano
+  #     - airflow
+  #   restart: unless-stopped
+  #
+  # airflow-metadata-db:
+  #   image: postgres
+  #   environment:
+  #     POSTGRES_PASSWORD: postgres # CHANGE ME
+  #     POSTGRES_DB: airflow
+  #     PGDATA: /var/lib/postgresql/data/pgdata
+  #   volumes:
+  #     - airflow_postgresql_data:/var/lib/postgresql/data
+  #   expose:
+  #     - 5432
+  #   networks:
+  #     - airflow
+  #   restart: unless-stopped
+
+networks:
+  meltano:
+  # # Uncomment if you are using the Airflow orchestrator, delete otherwise
+  # airflow:
+
+volumes:
+  meltano_postgresql_data:
+    driver: local
+  meltano_elt_logs_data:
+    driver: local
+  # # Uncomment if you are using the Airflow orchestrator, delete otherwise
+  # airflow_postgresql_data:
+  #   driver: local

docker-compose.yml

@@ -0,0 +1,33 @@
+x-meltano-image: &meltano-image
+  image: lakehouse/sarens-integration:latest
+
+services:
+  meltano:
+    <<: *meltano-image
+    command: dragon
+    restart: unless-stopped
+    networks:
+      - db_network
+
+  # # Uncomment if you are using the Airflow orchestrator, delete otherwise
+  # airflow-scheduler:
+  #   <<: *meltano-image
+  #   command: invoke airflow scheduler
+  #   expose:
+  #     - 8793
+  #   restart: unless-stopped
+  #
+  # airflow-webserver:
+  #   <<: *meltano-image
+  #   command: invoke airflow webserver
+  #   expose:
+  #     - 8080
+  #   ports:
+  #     - 8080:8080
+  #   restart: unless-stopped
+
+
+networks:
+  db_network:
+    external: true
+    name: db_network

plugins/files/files-docker--meltano.lock

@@ -0,0 +1,11 @@
+{
+  "plugin_type": "files",
+  "name": "files-docker",
+  "namespace": "files_docker",
+  "variant": "meltano",
+  "label": "Docker",
+  "docs": "https://hub.meltano.com/files/files-docker--meltano",
+  "repo": "https://github.com/meltano/files-docker",
+  "pip_url": "git+https://github.com/meltano/files-docker.git",
+  "logo_url": "https://hub.meltano.com/assets/logos/files/docker.png"
+}

plugins/files/files-docker-compose--meltano.lock

@@ -0,0 +1,11 @@
+{
+  "plugin_type": "files",
+  "name": "files-docker-compose",
+  "namespace": "files_docker_compose",
+  "variant": "meltano",
+  "label": "Docker Compose",
+  "docs": "https://hub.meltano.com/files/files-docker-compose--meltano",
+  "repo": "https://github.com/meltano/files-docker-compose",
+  "pip_url": "git+https://github.com/meltano/files-docker-compose.git",
+  "logo_url": "https://hub.meltano.com/assets/logos/files/docker-compose.png"
+}