Added docker-compose conf

erezsh · erezsh · commit 4534fff238b1 · 2022-12-15T17:59:22.000-03:00
diff --git a/dev/Dockerfile.prestosql.340 b/dev/Dockerfile.prestosql.340
@@ -0,0 +1,25 @@
+FROM openjdk:11-jdk-slim-buster
+
+ENV PRESTO_VERSION=340
+ENV PRESTO_SERVER_URL=https://repo1.maven.org/maven2/io/prestosql/presto-server/${PRESTO_VERSION}/presto-server-${PRESTO_VERSION}.tar.gz
+ENV PRESTO_CLI_URL=https://repo1.maven.org/maven2/io/prestosql/presto-cli/${PRESTO_VERSION}/presto-cli-${PRESTO_VERSION}-executable.jar
+ENV PRESTO_HOME=/opt/presto
+ENV PATH=${PRESTO_HOME}/bin:${PATH}
+
+WORKDIR $PRESTO_HOME
+
+RUN set -xe \
+    && apt-get update \
+    && apt-get install -y curl less python \
+    && curl -sSL $PRESTO_SERVER_URL | tar xz --strip 1 \
+    && curl -sSL $PRESTO_CLI_URL > ./bin/presto \
+    && chmod +x ./bin/presto \
+    && apt-get remove -y curl \
+    && rm -rf /var/lib/apt/lists/*
+
+VOLUME /data
+
+EXPOSE 8080
+
+ENTRYPOINT ["launcher"]
+CMD ["run"]
diff --git a/dev/dev.env b/dev/dev.env
@@ -0,0 +1,23 @@
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=Password1
+POSTGRES_DB=postgres
+
+MYSQL_DATABASE=mysql
+MYSQL_USER=mysql
+MYSQL_PASSWORD=Password1
+MYSQL_ROOT_PASSWORD=RootPassword1
+
+CLICKHOUSE_USER=clickhouse
+CLICKHOUSE_PASSWORD=Password1
+CLICKHOUSE_DB=clickhouse
+CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1
+
+# Vertica credentials
+APP_DB_USER=vertica
+APP_DB_PASSWORD=Password1
+VERTICA_DB_NAME=vertica
+
+# To prevent generating sample demo VMart data (more about it here https://www.vertica.com/docs/9.2.x/HTML/Content/Authoring/GettingStartedGuide/IntroducingVMart/IntroducingVMart.htm),
+# leave VMART_DIR and VMART_ETL_SCRIPT empty.
+VMART_DIR=
+VMART_ETL_SCRIPT=
diff --git a/dev/presto-conf/standalone/catalog/jmx.properties b/dev/presto-conf/standalone/catalog/jmx.properties
@@ -0,0 +1 @@
+connector.name=jmx
diff --git a/dev/presto-conf/standalone/catalog/memory.properties b/dev/presto-conf/standalone/catalog/memory.properties
@@ -0,0 +1 @@
+connector.name=memory
diff --git a/dev/presto-conf/standalone/catalog/postgresql.properties b/dev/presto-conf/standalone/catalog/postgresql.properties
@@ -0,0 +1,5 @@
+connector.name=postgresql
+connection-url=jdbc:postgresql://postgres:5432/postgres
+connection-user=postgres
+connection-password=Password1
+allow-drop-table=true
diff --git a/dev/presto-conf/standalone/catalog/tpcds.properties b/dev/presto-conf/standalone/catalog/tpcds.properties
@@ -0,0 +1 @@
+connector.name=tpcds
diff --git a/dev/presto-conf/standalone/catalog/tpch.properties b/dev/presto-conf/standalone/catalog/tpch.properties
@@ -0,0 +1 @@
+connector.name=tpch
diff --git a/dev/presto-conf/standalone/combined.pem b/dev/presto-conf/standalone/combined.pem
@@ -0,0 +1,52 @@
+-----BEGIN CERTIFICATE-----
+MIIERDCCAyygAwIBAgIUBxO/CflDP+0yZAXt5FKm/WQ4538wDQYJKoZIhvcNAQEL
+BQAwWTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
+GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDESMBAGA1UEAwwJbG9jYWxob3N0MB4X
+DTIyMDgyNDA4NTI1N1oXDTMyMDgyMTA4NTI1N1owWTELMAkGA1UEBhMCQVUxEzAR
+BgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5
+IEx0ZDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A
+MIIBCgKCAQEA3lNywkj/eGPGoFA3Lcx++98l17CRy+uzZMtJsr6lYAg1p/n1vPw0
+BQXI5TSBJ6vM/axtwgwrfXQsjQ/GYJKQkb6eEBCc3xb+Rk5HNBiBBZsIjYm0U1zz
+7dKnNwAznjx3j72s2ZQiqkoxcu7Bctw28ynbg0rjNkuUk3QESKuOgaTltpWKZiiu
+XwWasREeH6MH7ROy8db6cz+MwGaig0mUvGPmD97bPRD/X683RyOiXzEaogl/rpGK
+qZ3jRsmS8ZwawzKxx16kqPsX8/01EruGIoubMttr3YoZG044zq7nQqdAAz6wXx6V
+mgzToCHI+/g+8JS/bgqJTyb2Y6aGXExiuQIDAQABo4IBAjCB/zAdBgNVHQ4EFgQU
+5i1F8pTnwjFxw6W/0RjwpaJaK9MwgZYGA1UdIwSBjjCBi4AU5i1F8pTnwjFxw6W/
+0RjwpaJaK9OhXaRbMFkxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRl
+MSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxEjAQBgNVBAMMCWxv
+Y2FsaG9zdIIUBxO/CflDP+0yZAXt5FKm/WQ4538wDAYDVR0TBAUwAwEB/zALBgNV
+HQ8EBAMCAvwwFAYDVR0RBA0wC4IJbG9jYWxob3N0MBQGA1UdEgQNMAuCCWxvY2Fs
+aG9zdDANBgkqhkiG9w0BAQsFAAOCAQEAjBQLl/UFSd9TH2VLM1GH8bixtEJ9+rm7
+x+Jw665+XLjW107dJ33qxy9zjd3cZ2fynKg2Tb7+9QAvSlqpt2YMGP9jr4W2w16u
+ngbNB+kfoOotcUChk90aHmdHLZgOOve/ArFIvbr8douLOn0NAJBrj+iX4zC1pgEC
+9hsMUekkAPIcCGc0rEkEc8r8uiUBWNAdEWpBt0X2fE1ownLuB/E/+3HutLTw8Lv0
+b+jNt/vogVixcw/FF4atoO+F7S5FYzAb0U7YXaNISfVPVBsA89oPy7PlxULHDUIF
+Iq+vVqKdj1EXR+Iec0TMiMsa3MnIGkpL7ZuUXaG+xGBaVhGrUp67lQ==
+-----END CERTIFICATE-----
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEA3lNywkj/eGPGoFA3Lcx++98l17CRy+uzZMtJsr6lYAg1p/n1
+vPw0BQXI5TSBJ6vM/axtwgwrfXQsjQ/GYJKQkb6eEBCc3xb+Rk5HNBiBBZsIjYm0
+U1zz7dKnNwAznjx3j72s2ZQiqkoxcu7Bctw28ynbg0rjNkuUk3QESKuOgaTltpWK
+ZiiuXwWasREeH6MH7ROy8db6cz+MwGaig0mUvGPmD97bPRD/X683RyOiXzEaogl/
+rpGKqZ3jRsmS8ZwawzKxx16kqPsX8/01EruGIoubMttr3YoZG044zq7nQqdAAz6w
+Xx6VmgzToCHI+/g+8JS/bgqJTyb2Y6aGXExiuQIDAQABAoIBAD3pKwnjXhDeaA94
+hwUf7zSgfV9E8jTBHCGzYoB+CntljduLBd1stee4Jqt9JYIwm1MA00e4L9wtn8Jg
+ZDO8XLnZRRbgKW8ObhyR684cDMHM3GLdt/OG7P6LLLlqOvWTjQ/gF+Q3FjgplP+W
+cRRVMpAgVdqH3iHehi9RnWfHLlX3WkBC97SumWFzWqBnqUQAC8AvFHiUCqA9qIeA
+8ieOEoE17yv2nkmu+A5OZoCXtVfc2lQ90Fj9QZiv4rIVXBtTRRURJuvi2iX3nOPl
+MsjAUIBK1ndzpJ7wuLICSR1U3/npPC6Va06lTm0H/Q6DEqZjEHbx9TGY3pTgVXuA
++G0C5GkCgYEA/spvoDUMZrH2JE43TT/qMEHPtHW4qT6fTmzu08Gx++8nFmddNgSD
+zrdjxqPMUGV7Q5smwoHaQyqFxHMM2jh8icnV6VoBDrDdZM0eGFAs6HUjKmyaAdQO
+dC4kPiy3LX5pJUnQnmwq1fVsgXWGQF/LhD0L/y6xOiqdhZp/8nv6SFMCgYEA32GR
+gWJQdgWXTXxSEDn0twKPevgAT0s778/7h5osCLG82Q7ab2+Fc1qTleiwiQ2SAuOl
+mWvtz0Eg4dYe/q6jugqkEgAYZvdHGL7CSmC28O98fTLapgKQC5GUUan90sCbRec4
+kjbyx5scICNBYJVchdFg6UUSNz5czORUVgQEF0MCgYB1toUX2Spfj7yOTWyTTgIe
+RWl2kCS+XGYxT3aPcp+OK5E9cofH2xIiQOvh6+8K/beTJm0j0+ZIva6LcjPv5cTz
+y8H+S0zNwrymQ3Wx+eilhOi4QvBsA9KhrmekKfh/FjXxukadyo+HxhlZPjjGKPvX
+nnSacrICk4mvHhAasViSbQKBgD7mZGiAXJO/I0moVhtHlobp66j+qGerkacHc5ZN
+bVTNZ5XfPtbeGj/PI3u01/Dfp1u06m53G7GebznoZzXjyyqZ0HVZHYXw304yeNck
+wJ67cNx4M2VHl3QKfC86pMRxg8d9Qkq5ukdGf/b0tnYR2Mm9mYJV9rkjkFIJgU3v
+N4+tAoGABOlVGuRx2cSQ9QeC0AcqKlxXygdrzyadA7i0KNBZGGyrMSpJDrl2rrRn
+ylzAgGjvfilwQzZuqTm6Vo2yvaX+TTGS44B+DnxCZvuviftea++sNMjuEkBLTCpF
+xk2yOzsOnx652kWO4L+dVrDAxl65f3v0YaKWZI504LFYl18uS/E=
+-----END RSA PRIVATE KEY-----
diff --git a/dev/presto-conf/standalone/config.properties b/dev/presto-conf/standalone/config.properties
@@ -0,0 +1,8 @@
+coordinator=true
+node-scheduler.include-coordinator=true
+http-server.http.port=8080
+query.max-memory=5GB
+query.max-memory-per-node=1GB
+query.max-total-memory-per-node=2GB
+discovery-server.enabled=true
+discovery.uri=http://127.0.0.1:8080
diff --git a/dev/presto-conf/standalone/jvm.config b/dev/presto-conf/standalone/jvm.config
@@ -0,0 +1,9 @@
+-server
+-Xmx16G
+-XX:+UseG1GC
+-XX:G1HeapRegionSize=32M
+-XX:+UseGCOverheadLimit
+-XX:+ExplicitGCInvokesConcurrent
+-XX:+HeapDumpOnOutOfMemoryError
+-XX:+ExitOnOutOfMemoryError
+-XX:OnOutOfMemoryError=kill -9 %p
diff --git a/dev/presto-conf/standalone/log.properties b/dev/presto-conf/standalone/log.properties
@@ -0,0 +1 @@
+com.facebook.presto=INFO
diff --git a/dev/presto-conf/standalone/node.properties b/dev/presto-conf/standalone/node.properties
@@ -0,0 +1,3 @@
+node.environment=production
+node.data-dir=/data
+node.id=standalone
diff --git a/dev/presto-conf/standalone/password-authenticator.properties b/dev/presto-conf/standalone/password-authenticator.properties
@@ -0,0 +1,2 @@
+password-authenticator.name=file
+file.password-file=/opt/presto/etc/password.db
diff --git a/dev/presto-conf/standalone/password.db b/dev/presto-conf/standalone/password.db
@@ -0,0 +1 @@
+test:$2y$10$877iU3J5a26SPDjFSrrz2eFAq2DwMDsBAus92Dj0z5A5qNMNlnpHa
diff --git a/dev/trino-conf/etc/catalog/jms.properties b/dev/trino-conf/etc/catalog/jms.properties
@@ -0,0 +1 @@
+connector.name=jmx
diff --git a/dev/trino-conf/etc/catalog/memory.properties b/dev/trino-conf/etc/catalog/memory.properties
@@ -0,0 +1,2 @@
+connector.name=memory
+memory.max-data-per-node=128MB
diff --git a/dev/trino-conf/etc/catalog/postgresql.properties b/dev/trino-conf/etc/catalog/postgresql.properties
@@ -0,0 +1,4 @@
+connector.name=postgresql
+connection-url=jdbc:postgresql://postgres:5432/postgres
+connection-user=postgres
+connection-password=Password1
diff --git a/dev/trino-conf/etc/catalog/tpcds.properties b/dev/trino-conf/etc/catalog/tpcds.properties
@@ -0,0 +1 @@
+connector.name=tpcds
diff --git a/dev/trino-conf/etc/catalog/tpch.properties b/dev/trino-conf/etc/catalog/tpch.properties
@@ -0,0 +1 @@
+connector.name=tpch
diff --git a/dev/trino-conf/etc/config.properties b/dev/trino-conf/etc/config.properties
@@ -0,0 +1,5 @@
+coordinator=true
+node-scheduler.include-coordinator=true
+http-server.http.port=8080
+discovery.uri=http://localhost:8080
+discovery-server.enabled=true
diff --git a/dev/trino-conf/etc/jvm.config b/dev/trino-conf/etc/jvm.config
@@ -0,0 +1,12 @@
+-server
+-Xmx1G
+-XX:-UseBiasedLocking
+-XX:+UseG1GC
+-XX:G1HeapRegionSize=32M
+-XX:+ExplicitGCInvokesConcurrent
+-XX:+HeapDumpOnOutOfMemoryError
+-XX:+UseGCOverheadLimit
+-XX:+ExitOnOutOfMemoryError
+-XX:ReservedCodeCacheSize=256M
+-Djdk.attach.allowAttachSelf=true
+-Djdk.nio.maxCachedBufferSize=2000000
diff --git a/dev/trino-conf/etc/node.properties b/dev/trino-conf/etc/node.properties
@@ -0,0 +1,3 @@
+node.environment=docker
+node.data-dir=/data/trino
+plugin.dir=/usr/lib/trino/plugin
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -0,0 +1,130 @@
+version: "3.8"
+
+services:
+    postgres:
+      container_name: dd-postgresql
+      image: postgres:14.1-alpine
+      # work_mem: less tmp files
+      # maintenance_work_mem: improve table-level op perf
+      # max_wal_size: allow more time before merging to heap
+      command: >
+        -c work_mem=1GB
+        -c maintenance_work_mem=1GB
+        -c max_wal_size=8GB
+      restart: always
+      volumes:
+        - postgresql-data:/var/lib/postgresql/data:delegated
+      ports:
+        - '5432:5432'
+      expose:
+        - '5432'
+      env_file:
+        - dev/dev.env
+      tty: true
+      networks:
+        - local
+
+    mysql:
+      container_name: dd-mysql
+      image: mysql:oracle
+      # fsync less aggressively for insertion perf for test setup
+      command: >
+        --default-authentication-plugin=mysql_native_password
+        --binlog-cache-size=16M
+        --key_buffer_size=0
+        --max_connections=1000
+        --innodb_flush_log_at_trx_commit=2
+        --innodb_flush_log_at_timeout=10
+        --innodb_log_compressed_pages=OFF
+        --sync_binlog=0
+      restart: always
+      volumes:
+        - mysql-data:/var/lib/mysql:delegated
+      user: mysql
+      ports:
+        - '3306:3306'
+      expose:
+        - '3306'
+      env_file:
+        - dev/dev.env
+      tty: true
+      networks:
+        - local
+
+    clickhouse:
+      container_name: dd-clickhouse
+      image: clickhouse/clickhouse-server:21.12.3.32
+      restart: always
+      volumes:
+          - clickhouse-data:/var/lib/clickhouse:delegated
+      ulimits:
+        nproc: 65535
+        nofile:
+          soft: 262144
+          hard: 262144
+      ports:
+        - '8123:8123'
+        - '9000:9000'
+      expose:
+        - '8123'
+        - '9000'
+      env_file:
+        - dev/dev.env
+      tty: true
+      networks:
+      - local
+
+    # prestodb.dbapi.connect(host="127.0.0.1", user="presto").cursor().execute('SELECT * FROM system.runtime.nodes')
+    presto:
+        container_name: dd-presto
+        build:
+            context: ./dev
+            dockerfile: ./Dockerfile.prestosql.340
+        volumes:
+            - ./dev/presto-conf/standalone:/opt/presto/etc:ro
+        ports:
+            - '8080:8080'
+        tty: true
+        networks:
+            - local
+
+    trino:
+        container_name: dd-trino
+        image: 'trinodb/trino:389'
+        hostname: trino
+        ports:
+            - '8081:8080'
+        volumes:
+            - ./dev/trino-conf/etc:/etc/trino:ro
+        networks:
+            - local
+
+    vertica:
+      container_name: dd-vertica
+      image: vertica/vertica-ce:12.0.0-0
+      restart: always
+      volumes:
+          - vertica-data:/data:delegated
+      ports:
+        - '5433:5433'
+        - '5444:5444'
+      expose:
+        - '5433'
+        - '5444'
+      env_file:
+        - dev/dev.env
+      tty: true
+      networks:
+      - local
+
+
+
+volumes:
+  postgresql-data:
+  mysql-data:
+  clickhouse-data:
+  vertica-data:
+
+networks:
+  local:
+    driver: bridge
