csghub-dataflow/Dockerfile-argo at main · OpenCSGs/csghub-dataflow · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
# Argo Pod 全量算子镜像：完整 dataflow 依赖 + 仅复制运行代码
# 构建: ./scripts/build-push-argo.sh [registry] [tag]

ARG PYTHON_IMAGE=docker.io/python:3.10.14
FROM ${PYTHON_IMAGE}

# PySpark 工具任务（如 quality_classifier）需要 JVM
WORKDIR /opt
RUN wget https://aka.ms/download-jdk/microsoft-jdk-17.0.9-linux-x64.tar.gz -O jdk.tar.gz && \
    tar -xzf jdk.tar.gz && \
    rm -rf jdk.tar.gz && \
    mv jdk-17.0.9+8 jdk

ENV JAVA_HOME=/opt/jdk
ENV PATH="${JAVA_HOME}/bin:${PATH}"

WORKDIR /dataflow
ENV DATA_DIR=/dataflow_data \
    PIP_DISABLE_PIP_VERSION_CHECK=1 \
    PYTHONDONTWRITEBYTECODE=1 \
    PYTHONUNBUFFERED=1

ARG BUILD_CN=false
ARG PRELOAD_ASSETS=true
ARG PIP_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple/

# 系统库：DB、OpenCV、git-lfs、kenlm/fasttext 等原生扩展编译
RUN if [ "$BUILD_CN" = "true" ]; then \
      rm -rf /etc/apt/sources.list.d/debian.sources || true; \
      echo "deb http://mirrors.aliyun.com/debian bookworm main contrib non-free" > /etc/apt/sources.list; \
      echo "deb http://mirrors.aliyun.com/debian-security bookworm-security main contrib non-free" >> /etc/apt/sources.list; \
      echo "deb http://mirrors.aliyun.com/debian bookworm-updates main contrib non-free" >> /etc/apt/sources.list; \
    fi && \
    apt-get update && \
    apt-get install --no-install-recommends -y \
      build-essential \
      libpq-dev \
      libgl1-mesa-glx \
      libsndfile1 \
      git-lfs \
      wget \
      ca-certificates && \
    apt-get clean && rm -rf /var/lib/apt/lists/* && \
    git lfs install

# 全量 Python 依赖（与 docker/dataflow_requirements.txt 一致，覆盖全部算子）
COPY docker/dataflow_requirements.txt /tmp/docker/dataflow_requirements.txt

RUN --mount=type=cache,target=/root/.cache/pip \
    if [ "$BUILD_CN" = "true" ]; then \
      pip install --no-cache-dir -r /tmp/docker/dataflow_requirements.txt -i "${PIP_INDEX_URL}"; \
    else \
      pip install --no-cache-dir -r /tmp/docker/dataflow_requirements.txt; \
    fi

RUN git config --global user.email "dataflow@opencsg.com" && \
    git config --global user.name "dataflow" && \
    git config --global --add safe.directory '*'

RUN if [ "$PRELOAD_ASSETS" = "true" ]; then \
      mkdir -p /root/.cache/data_engine/assets /root/.cache/data_engine/models && \
      wget -q -O /root/.cache/data_engine/assets/flagged_words.json \
        https://dail-wlcb.oss-cn-wulanchabu.aliyuncs.com/data_juicer/flagged_words.json && \
      wget -q -O /root/.cache/data_engine/assets/stopwords.json \
        https://dail-wlcb.oss-cn-wulanchabu.aliyuncs.com/data_juicer/stopwords.json && \
      wget -q -O /root/.cache/data_engine/models/zh.sp.model \
        https://dail-wlcb.oss-cn-wulanchabu.aliyuncs.com/data_juicer/models/zh.sp.model && \
      wget -q -O /root/.cache/data_engine/models/en.sp.model \
        https://dail-wlcb.oss-cn-wulanchabu.aliyuncs.com/data_juicer/models/en.sp.model; \
    fi

# 运行代码（不 COPY 整仓 demos/docs 等，见 .dockerignore）
# setup.py 供 data_server.utils.project_paths.get_project_root() 定位项目根（runtime_logs 等路径）
COPY setup.py ./
COPY run_dataflow_task.py ./
COPY data_server ./data_server
COPY data_engine ./data_engine
COPY configs ./configs

CMD ["python", "run_dataflow_task.py", "--help"]