Skip to content

Commit cd544ff

Browse files
tuhaihetuhaihe
committed
Release: Use pip3 download for Python packages
Replace bundled Python source packages with pip3 download to comply with Apache Release policy. - Remove 7 bundled tar.gz packages from gpMgmt/bin/pythonSrc/ext/ - Add download-python-deps target using pip3 download - Update build targets to depend on download step - Preserve original compilation logic and --with-pythonsrc-ext functionality - Update LICENSE, python-dependencies.txt, and Windows build script - Add .gitignore rules for downloaded files This maintains the same build behavior while ensuring Apache compliance by removing third-party packages from source tree. See: #961
1 parent de84d5a commit cd544ff

21 files changed

+37
-214
lines changed

‎LICENSE‎

Lines changed: 0 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -218,11 +218,7 @@ The PostgreSQL software includes:
218218
src/bin/pg_controldata/pg_controldata.c
219219
copyright (c) Oliver Elphick <olly@lfix.co.uk>, 2001
220220

221-
gpMgmt/bin/pythonSrc/ext/behave-1.2.6.tar.gz
222-
see licenses/LICENSE-bsd2-behave.txt
223221

224-
gpMgmt/bin/pythonSrc/ext/mock-1.0.1.tar.gz
225-
see licenses/LICENSE-bsd2-mock.txt
226222

227223
----------------------------
228224
BSD-4-Clause (University of California-Specific)
@@ -250,9 +246,6 @@ The PostgreSQL software includes:
250246
src/backend/libpq/sha2.h
251247
see licenses/LICENSE-bsd3-sha2.txt
252248

253-
gpMgmt/bin/pythonSrc/ext/psutil-5.7.0.tar.gz
254-
see licenses/LICENSE-bsd3-psutil.txt
255-
256249
contrib/pax_storage/src/cpp/contrib/googletest
257250
see licenses/LICENSE-googletest.txt
258251

@@ -262,15 +255,6 @@ The PostgreSQL software includes:
262255
----------------------------
263256
MIT LICENSE
264257

265-
gpMgmt/bin/pythonSrc/ext/PyYAML-5.4.1.tar.gz
266-
see licenses/LICENSE-pyyaml.txt
267-
268-
gpMgmt/bin/pythonSrc/ext/setuptools-36.6.0.tar.gz
269-
see licenses/LICENSE-setuptools.txt
270-
271-
gpMgmt/bin/pythonSrc/ext/parse-1.8.2.tar.gz
272-
see licenses/LICENSE-parse.txt
273-
274258
dependencies/yyjson
275259
see licenses/LICENSE-yyjson.txt
276260

@@ -312,21 +296,18 @@ The PostgreSQL software includes:
312296
test-ctype LICENSE
313297

314298
src/test/locale/test-ctype.c
315-
316299
see licenses/LICENSE-test-ctype.txt
317300

318301
----------------------------
319302
regex LICENSE
320303

321304
src/backend/regex
322-
323305
see licenses/LICENSE-regex.txt
324306

325307
----------------------------
326308
PostgreSQL-style License
327309

328310
src/port/gettimeofday.c
329-
330311
see licenses/LICENSE-gettimeofday.txt
331312

332313
================================================================================
@@ -349,22 +330,14 @@ The Greenplum Database software includes:
349330
Stream License
350331

351332
gpMgmt/bin/stream/*
352-
353333
see licenses/LICENSE-stream.txt
354334

355335
----------------------------
356336
0-clause license ("Zero Clause BSD")
357337

358338
gpcontrib/orafce/*
359-
360339
see licenses/LICENSE-orafce.txt
361340

362-
================================================================================
363-
PostgreSQL-style license
364-
365-
gpMgmt/bin/pythonSrc/ext/PyGreSQL-5.2.tar.gz
366-
see licenses/LICENSE-pygresql.txt
367-
368341
================================================================================
369342
This product includes code from pg_cron under PostgreSQL license:
370343

‎gpAux/client/install/src/windows/CreatePackage.bat‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,8 @@ type nul > %GPDB_INSTALL_PATH%\bin\gppylib\__init__.py
1111
copy ..\..\..\..\..\gpMgmt\bin\gppylib\gpversion.py %GPDB_INSTALL_PATH%\bin\gppylib\
1212
perl -pi.bak -e "s,\$Revision\$,%VERSION%," %GPDB_INSTALL_PATH%\bin\gpload.py
1313
copy ..\..\..\..\..\gpMgmt\bin\gpload.bat %GPDB_INSTALL_PATH%\bin
14-
for %%f in (..\..\..\..\..\gpMgmt\bin\pythonSrc\ext\PyYAML-*.tar.gz) do tar -xf %%f
15-
for /D %%d in (PyYAML-*) do copy %%d\lib\yaml\* %GPDB_INSTALL_PATH%\lib\python\yaml
14+
REM Install PyYAML using pip instead of extracting from tarball
15+
pip3 install --target=%GPDB_INSTALL_PATH%\lib\python PyYAML==5.4.1
1616
perl -p -e "s,__VERSION_PLACEHOLDER__,%VERSION%," greenplum-clients.wxs > greenplum-clients-%VERSION%.wxs
1717
candle.exe -nologo greenplum-clients-%VERSION%.wxs -out greenplum-clients-%VERSION%.wixobj -dSRCDIR=%GPDB_INSTALL_PATH% -dVERSION=%VERSION%
1818
light.exe -nologo -sval greenplum-clients-%VERSION%.wixobj -out greenplum-clients-x86_64.msi

‎gpMgmt/bin/.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,3 +27,5 @@
2727
# ignore anything in ext but keep folder
2828
ext/*
2929
!ext/.gitkeep
30+
# Python sources and temporary install dirs
31+
pythonSrc/ext/*

‎gpMgmt/bin/Makefile‎

Lines changed: 29 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -79,26 +79,41 @@ endif
7979
#
8080
# Python Libraries
8181
#
82+
# Download Python source packages using pip3 download
83+
# This replaces the previously bundled tar.gz files to comply with Apache Release policy
84+
#
85+
MOCK_VERSION=1.0.1
86+
PYGRESQL_VERSION=5.2
87+
PSUTIL_VERSION=5.7.0
88+
PYYAML_VERSION=5.4.1
89+
90+
download-python-deps:
91+
@echo "--- Downloading Python dependencies for gpMgmt modules"
92+
@mkdir -p $(PYLIB_SRC_EXT)
93+
# Download psutil
94+
pip3 download --no-deps --no-binary :all: --no-build-isolation psutil==$(PSUTIL_VERSION) --dest $(PYLIB_SRC_EXT)/
95+
# Download PyYAML, wheel is for metadata vefiry;
96+
# cython is used for building, see https://github.com/apache/cloudberry/issues/1201
97+
pip3 install wheel "cython<3.0.0"
98+
pip3 download --no-deps --no-binary :all: --no-build-isolation PyYAML==$(PYYAML_VERSION) --dest $(PYLIB_SRC_EXT)/
99+
# Download PyGreSQL, the `PATH` is needed to avoid cannot find the pg_config
100+
PATH=$(DESTDIR)$(bindir):$$PATH pip3 download --no-deps --no-binary :all: --no-build-isolation PyGreSQL==$(PYGRESQL_VERSION) --dest $(PYLIB_SRC_EXT)/
82101

83102
#
84103
# PyGreSQL
85104
#
86-
PYGRESQL_VERSION=5.2
87105
PYGRESQL_DIR=PyGreSQL-$(PYGRESQL_VERSION)
88-
pygresql:
106+
pygresql: download-python-deps
89107
@echo "--- PyGreSQL"
90108
cd $(PYLIB_SRC_EXT)/ && $(TAR) xzf $(PYGRESQL_DIR).tar.gz
91109
cd $(PYLIB_SRC_EXT)/$(PYGRESQL_DIR)/ && PATH=$(DESTDIR)$(bindir):$$PATH LDFLAGS='$(LDFLAGS) $(PYGRESQL_LDFLAGS)' python3 setup.py build
92110
cp -r $(PYLIB_SRC_EXT)/$(PYGRESQL_DIR)/build/lib*-3*/* $(PYLIB_DIR)/
93111

94-
95112
#
96113
# PSUTIL
97114
#
98-
PSUTIL_VERSION=5.7.0
99115
PSUTIL_DIR=psutil-$(PSUTIL_VERSION)
100-
101-
psutil:
116+
psutil: download-python-deps
102117
@echo "--- psutil"
103118
ifeq "$(findstring $(BLD_ARCH),aix7_ppc_64 )" ""
104119
cd $(PYLIB_SRC_EXT)/ && $(TAR) xzf $(PSUTIL_DIR).tar.gz
@@ -109,24 +124,18 @@ endif
109124
#
110125
# PYYAML
111126
#
112-
PYYAML_VERSION=5.4.1
113127
PYYAML_DIR=PyYAML-$(PYYAML_VERSION)
114128

115-
pyyaml:
129+
pyyaml: download-python-deps
116130
@echo "--- pyyaml"
117131
cd $(PYLIB_SRC_EXT)/ && $(TAR) xzf $(PYYAML_DIR).tar.gz
118-
cd $(PYLIB_SRC_EXT)/$(PYYAML_DIR)/ && env -u CC python3 setup.py build
132+
cd $(PYLIB_SRC_EXT)/$(PYYAML_DIR)/ && env -u CC CFLAGS="-w" python3 setup.py build
119133
cp -r $(PYLIB_SRC_EXT)/$(PYYAML_DIR)/build/lib*-3*/* $(PYLIB_DIR)/
120134

121135
#
122136
# MOCK SETUP
123137
#
124-
MOCK_VERSION=1.0.1
125138
MOCK_DIR=mock-$(MOCK_VERSION)
126-
SETUP_TOOLS_VERSION=36.6.0
127-
PARSE_VERSION=1.8.2
128-
SETUP_TOOLS_DIR=setuptools-$(SETUP_TOOLS_VERSION)
129-
PARSE_DIR=parse-$(PARSE_VERSION)
130139
PYTHONSRC_INSTALL=$(PYLIB_SRC_EXT)/install
131140
PYTHON_VERSION=$(shell python3 -c "import sys; print ('%s.%s' % (sys.version_info[0:2]))")
132141
PYTHONSRC_INSTALL_SITE=$(PYLIB_SRC_EXT)/install/lib/python$(PYTHON_VERSION)/site-packages
@@ -140,8 +149,10 @@ $(MOCK_BIN):
140149
@if [ "$(UBUNTU_PLATFORM)" = "Ubuntu" ]; then\
141150
pip3 install mock;\
142151
else\
143-
mkdir -p $(PYTHONSRC_INSTALL_SITE) && \
144-
cd $(PYLIB_SRC_EXT)/ && $(TAR) xzf $(MOCK_DIR).tar.gz && \
152+
mkdir -p $(PYLIB_SRC_EXT) && \
153+
pip3 download --no-deps --no-binary :all: --no-build-isolation mock==$(MOCK_VERSION) --dest $(PYLIB_SRC_EXT)/ && \
154+
mkdir -p $(PYTHONSRC_INSTALL_SITE) && \
155+
cd $(PYLIB_SRC_EXT)/ && unzip -q $(MOCK_DIR).zip && \
145156
cd $(PYLIB_SRC_EXT)/$(MOCK_DIR)/ && \
146157
PYTHONPATH=$(PYTHONSRC_INSTALL_PYTHON_PATH) python3 setup.py install --prefix $(PYTHONSRC_INSTALL) ; \
147158
fi;
@@ -207,8 +218,8 @@ installcheck: installcheck-bash
207218

208219
clean distclean:
209220
rm -rf $(RUFF_BIN) $(SRC)/ruff.txt $(SRC)/.ruff_cache
210-
rm -rf $(PYLIB_SRC_EXT)/$(PYGRESQL_DIR)/build
211-
rm -rf $(PYLIB_SRC)/$(PYGRESQL_DIR)/build
221+
rm -rf $(PYLIB_SRC_EXT)/*.tar.gz $(PYLIB_SRC_EXT)/*.zip $(PYLIB_SRC_EXT)/*.whl
222+
rm -rf $(PYLIB_SRC_EXT)/*/
212223
rm -rf *.pyc
213224
rm -f analyzedbc gpactivatestandbyc gpaddmirrorsc gpcheckcatc \
214225
gpcheckperfc gpcheckresgroupimplc gpchecksubnetcfgc gpconfigc \

‎gpMgmt/bin/pythonSrc/ext/.gitkeep‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# This directory is used for downloading Python source packages via pip3 download
2+
# The downloaded tar.gz files should not be committed to git

‎gpMgmt/bin/pythonSrc/ext/PyGreSQL-5.2.tar.gz‎

-261 KB
Binary file not shown.

‎gpMgmt/bin/pythonSrc/ext/PyYAML-5.4.1.tar.gz‎

-176 KB
Binary file not shown.

‎gpMgmt/bin/pythonSrc/ext/behave-1.2.6.tar.gz‎

-4.35 MB
Binary file not shown.

‎gpMgmt/bin/pythonSrc/ext/mock-1.0.1.tar.gz‎

-799 KB
Binary file not shown.

‎gpMgmt/bin/pythonSrc/ext/parse-1.8.2.tar.gz‎

-27.3 KB
Binary file not shown.

0 commit comments

Comments
 (0)