diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..df598c6 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +# Ignore appsmith container volumes/data +docker/appsmith/stacks/ + +# Ignore other things that don't need to be in the Docker image +.git/ +venv/ +__pycache__/ +*.pyc \ No newline at end of file diff --git a/docker/docker-compose-with-airflow.yml b/docker/docker-compose-with-airflow.yml index 03bce4f..9b3d1f1 100644 --- a/docker/docker-compose-with-airflow.yml +++ b/docker/docker-compose-with-airflow.yml @@ -1,5 +1,5 @@ #version: '3.1' -version: '2.1' +# version: '2.1' # # https://airflow.apache.org/docs/apache-airflow/2.2.1/start/docker.html @@ -44,7 +44,8 @@ version: '2.1' x-airflow-common: &airflow-common - image: ${AIRFLOW_IMAGE_NAME:-apache/airflow:2.2.4} + # image: ${AIRFLOW_IMAGE_NAME:-apache/airflow:2.2.4} + image: apache/airflow:2.2.4-python3.8 environment: &airflow-common-env AIRFLOW__CORE__EXECUTOR: CeleryExecutor @@ -53,7 +54,7 @@ x-airflow-common: AIRFLOW__CELERY__BROKER_URL: redis://:@redis:6379/0 AIRFLOW__CORE__FERNET_KEY: '' AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true' - AIRFLOW__CORE__LOAD_EXAMPLES: 'true' + # AIRFLOW__CORE__LOAD_EXAMPLES: 'true' AIRFLOW__CORE__ENABLE_XCOM_PICKLING: 'true' PYTHON_BASE_IMAGE: "python:3:8-slim-buster" POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} @@ -91,6 +92,15 @@ services: - .:/db_dumps - ../db/dot:/docker-entrypoint-initdb.d + appsmith: + image: index.docker.io/appsmith/appsmith-ce + container_name: appsmith + ports: + - "82:80" + - "446:443" + volumes: + - ./appsmith/stacks:/appsmith-stacks + # Not needed here, because we deploy DOT to airflow worker #dot: # build: @@ -188,6 +198,10 @@ services: airflow-worker: <<: *airflow-common + image: dot-airflow-worker:latest + build: + context: .. + dockerfile: ./docker/dot/Dockerfile.airflow command: celery worker restart: always diff --git a/docker/dot/Dockerfile b/docker/dot/Dockerfile index 686e1f4..bce7a74 100644 --- a/docker/dot/Dockerfile +++ b/docker/dot/Dockerfile @@ -1,8 +1,16 @@ FROM python:3.8-slim-buster WORKDIR app/ +# Switch to the root user to install system-level dependencies if needed +USER root -RUN apt-get update +RUN sed -i 's|deb.debian.org|archive.debian.org|g' /etc/apt/sources.list \ + && sed -i 's|security.debian.org|archive.debian.org|g' /etc/apt/sources.list \ + && sed -i '/buster-updates/d' /etc/apt/sources.list \ + && printf 'Acquire::Check-Valid-Until "false";\n' > /etc/apt/apt.conf.d/99no-check-valid-until \ + && rm -f /etc/apt/sources.list.d/pgdg.list \ + && rm -f /etc/apt/sources.list.d/mssql-release.list \ + && apt-get update RUN pip install --upgrade pip diff --git a/docker/dot/Dockerfile.airflow b/docker/dot/Dockerfile.airflow new file mode 100755 index 0000000..8d6ba80 --- /dev/null +++ b/docker/dot/Dockerfile.airflow @@ -0,0 +1,26 @@ +# Start with the exact same Airflow image from Docker Hub +FROM apache/airflow:2.2.4-python3.8 + +# Switch to the root user to install system-level dependencies +USER root + +RUN sed -i 's|deb.debian.org|archive.debian.org|g' /etc/apt/sources.list \ + && sed -i 's|security.debian.org|archive.debian.org|g' /etc/apt/sources.list \ + && sed -i '/buster-updates/d' /etc/apt/sources.list \ + && printf 'Acquire::Check-Valid-Until "false";\n' > /etc/apt/apt.conf.d/99no-check-valid-until \ + && rm -f /etc/apt/sources.list.d/pgdg.list \ + && rm -f /etc/apt/sources.list.d/mssql-release.list \ + && apt-get update + +RUN apt-get -y install postgresql-client + + +# Switch back to the 'airflow' user (best practice for Airflow images) +USER airflow + +# Copy DOT requirements and install script +COPY --chown=airflow:root ./dot/requirements_dot.txt /tmp/requirements_dot.txt +COPY --chown=airflow:root ./dot/install_dot.sh /tmp/install_dot.sh + +# Run install script +RUN chmod +x /tmp/install_dot.sh && cd /tmp && ./install_dot.sh diff --git a/dot/install_dot.sh b/dot/install_dot.sh index 5346d9d..160227d 100755 --- a/dot/install_dot.sh +++ b/dot/install_dot.sh @@ -1,7 +1,19 @@ +#!/usr/bin/env bash # Simple install script to install Python dependencies, dbt and ge +set -euo pipefail # Python packages -pip install --upgrade pip +# pip install --upgrade pip +pip install "pip<22" \ + "setuptools>=59.1.1,<59.7.0" \ + "wheel<0.45" \ + "packaging<22" + # Based on this https://stackoverflow.com/questions/69287269/installing-ruamel-yaml-clib-with-docker. -pip install -U pip setuptools wheel ruamel.yaml ruamel.yaml.clib==0.2.6 -pip install -r requirements_dot.txt \ No newline at end of file +# pip install -U pip setuptools wheel ruamel.yaml ruamel.yaml.clib==0.2.6 +pip install "ruamel.yaml" "ruamel.yaml.clib==0.2.6" + +pip install -r requirements_dot.txt + +# sanity check +pip check \ No newline at end of file diff --git a/dot/requirements_dot.txt b/dot/requirements_dot.txt old mode 100644 new mode 100755 index 90222a2..0bde6a3 --- a/dot/requirements_dot.txt +++ b/dot/requirements_dot.txt @@ -2,16 +2,19 @@ black==21.12b0 dbt-core==1.0.0 dbt-extractor==0.4.0 dbt-postgres==1.0.0 -decorator==5.1.0 +# decorator==5.1.0 +decorator==4.4.2 Faker==15.1.1 faker_airtravel==0.4 flatten_json==0.1.13 gdown==4.5.3 -great-expectations==0.12.10 +# great-expectations==0.12.10 +great-expectations==0.14.13 ipython==8.10.0 ipython-genutils==0.2.0 jinja2==2.11.3 json-rpc==1.13.0 +jsonschema==3.1.1 jupyter-client==7.1.0 jupyter-core==4.9.1 jupyterlab-pygments==0.1.2 @@ -24,23 +27,34 @@ nest-asyncio==1.5.4 notebook==6.4.12 numpy==1.22.0 oauthlib==3.2.2 +openapi-spec-validator==0.2.9 openpyxl==3.0.9 pandas==1.3.4 -proto-plus==1.19.8 +packaging<22 +pip<22 +# proto-plus==1.19.8 +proto-plus==1.18.1 psycopg2-binary==2.9.2 +pydantic==1.9.2 Pygments==2.10.0 pylint==2.14.1 pytest==7.2.0 pytest-timeout==2.1.0 python_on_whales==0.53.0 -python-slugify==5.0.2 -PyYAML==6.0 -requests==2.23.0 +# python-slugify==5.0.2 +python-slugify>=3.0.0,<5.0 +# PyYAML==6.0 +PyYAML>=5.1,<6 +# requests==2.23.0 +requests>=2.26.0,<3 requests-oauthlib==1.3.0 rsa==4.8 ruamel.yaml==0.17.17 ruamel.yaml.clib==0.2.6 -setuptools-scm==6.3.2 +setuptools>=59.1.1,<59.7.0 +# setuptools-scm==6.3.2 +setuptools-scm<6 SQLAlchemy==1.3.24 toolz==0.11.2 typing-extensions==3.10.0.2 +wheel<0.45