Skip to content
This repository was archived by the owner on Feb 2, 2024. It is now read-only.

Commit 63cc4e2

Browse files
Vyacheslav-Smirnovshssf
Vyacheslav-Smirnov
authored and
shssf
committed
Numba master build (#298)
* Add Numba recipe; --use-numba-master to build and test with it * Change Travis and Azure configs * Change config for Azure * Render environment after Numba build * Add allowfailure to Numba master builds; fix conde-style * Remove prefix limiting; try numpy 1.17 for Numba master * Add tests launch for Numba master build * Fix numba_master_channel for Windows in test.py
1 parent 0369309 commit 63cc4e2

16 files changed

Lines changed: 594 additions & 47 deletions

‎.travis.yml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ env:
1111
- CONDA_PREFIX_PATH=$HOME/miniconda3
1212
matrix:
1313
- BUILD_MODE=package TEST_MODE=conda RUN_COVERAGE=False SDC_NP_MPI=1
14-
- BUILD_MODE=package TEST_MODE=conda RUN_COVERAGE=False SDC_NP_MPI=2
1514
- BUILD_MODE=develop TEST_MODE=develop RUN_COVERAGE=True
1615

1716
before_install:

‎azure-pipelines.yml‎

Lines changed: 40 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,66 @@
11
jobs:
22
- template: buildscripts/azure/template-windows.yml
33
parameters:
4-
name: Windows_2016
4+
name: Windows2016_vs2017_Numba046
55
vmImage: vs2017-win2016
6+
allowFailure: false
67
matrix:
7-
vs2017_py3.7_numpy1.16_np0:
8+
py3.7_numpy1.16:
89
PYTHON_VER: '3.7'
910
NUMPY_VER: '1.16'
11+
USE_NUMBA_MASTER: 'False'
1012

11-
vs2017_py3.7_numpy1.16_np2:
12-
SDC_NP_MPI: '2'
13+
- template: buildscripts/azure/template-linux-macos.yml
14+
parameters:
15+
name: macOS1013_xcode9_Numba046
16+
vmImage: xcode9-macos10.13
17+
allowFailure: false
18+
matrix:
19+
py3.7_numpy1.16:
1320
PYTHON_VER: '3.7'
1421
NUMPY_VER: '1.16'
22+
USE_NUMBA_MASTER: 'False'
1523

1624
- template: buildscripts/azure/template-linux-macos.yml
1725
parameters:
18-
name: OSX_10_13
19-
vmImage: xcode9-macos10.13
26+
name: Ubuntu1604_Numba046
27+
vmImage: ubuntu-16.04
28+
allowFailure: false
2029
matrix:
21-
xcode9_py3.7_numpy1.16_np0:
30+
py3.7_numpy1.16:
2231
PYTHON_VER: '3.7'
2332
NUMPY_VER: '1.16'
33+
USE_NUMBA_MASTER: 'False'
2434

25-
xcode9_py3.7_numpy1.16_np2:
26-
SDC_NP_MPI: '2'
35+
- template: buildscripts/azure/template-windows.yml
36+
parameters:
37+
name: Windows2016_vs2017_NumbaMaster
38+
vmImage: vs2017-win2016
39+
allowFailure: true
40+
matrix:
41+
py3.7_numpy1.16:
42+
PYTHON_VER: '3.7'
43+
NUMPY_VER: '1.16'
44+
USE_NUMBA_MASTER: 'True'
45+
46+
- template: buildscripts/azure/template-linux-macos.yml
47+
parameters:
48+
name: macOS1013_xcode9_NumbaMaster
49+
vmImage: xcode9-macos10.13
50+
allowFailure: true
51+
matrix:
52+
py3.7_numpy1.16:
2753
PYTHON_VER: '3.7'
2854
NUMPY_VER: '1.16'
55+
USE_NUMBA_MASTER: 'True'
2956

3057
- template: buildscripts/azure/template-linux-macos.yml
3158
parameters:
32-
name: Ubuntu_16_04
59+
name: Ubuntu1604_NumbaMaster
3360
vmImage: ubuntu-16.04
61+
allowFailure: true
3462
matrix:
35-
py3.7_numpy1.16_np3:
36-
SDC_NP_MPI: '3'
63+
py3.7_numpy1.16:
3764
PYTHON_VER: '3.7'
3865
NUMPY_VER: '1.16'
66+
USE_NUMBA_MASTER: 'True'

‎buildscripts/azure/template-linux-macos.yml‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
parameters:
22
name: ''
33
vmImage: ''
4+
allowFailure: ''
45
matrix: []
56

67
jobs:
@@ -20,9 +21,19 @@ jobs:
2021
displayName: 'Install conda'
2122
2223
- script: |
23-
$HOME/miniconda3/bin/python buildscripts/build.py --build-mode=package --python=$PYTHON_VER --numpy=$NUMPY_VER --conda-prefix=$HOME/miniconda3
24+
if [ "$USE_NUMBA_MASTER" == "False" ]; then
25+
$HOME/miniconda3/bin/python buildscripts/build.py --build-mode=package --python=$PYTHON_VER --numpy=$NUMPY_VER --conda-prefix=$HOME/miniconda3;
26+
else
27+
$HOME/miniconda3/bin/python buildscripts/build.py --build-mode=package --python=$PYTHON_VER --numpy=$NUMPY_VER --use-numba-master --conda-prefix=$HOME/miniconda3;
28+
fi
2429
displayName: 'Build conda and wheel packages'
30+
continueOnError: ${{ parameters.allowFailure }}
2531
2632
- script: |
27-
$HOME/miniconda3/bin/python buildscripts/test.py --test-mode=conda --python=$PYTHON_VER --numpy=$NUMPY_VER --conda-prefix=$HOME/miniconda3
33+
if [ "$USE_NUMBA_MASTER" == "False" ]; then
34+
$HOME/miniconda3/bin/python buildscripts/test.py --test-mode=conda --python=$PYTHON_VER --numpy=$NUMPY_VER --conda-prefix=$HOME/miniconda3;
35+
else
36+
$HOME/miniconda3/bin/python buildscripts/test.py --test-mode=conda --python=$PYTHON_VER --numpy=$NUMPY_VER --use-numba-master --conda-prefix=$HOME/miniconda3;
37+
fi
2838
displayName: 'Test conda package'
39+
continueOnError: ${{ parameters.allowFailure }}

‎buildscripts/azure/template-windows.yml‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
parameters:
22
name: ''
33
vmImage: ''
4+
allowFailure: ''
45
matrix: []
56

67
jobs:
@@ -19,9 +20,17 @@ jobs:
1920
# displayName: 'Install conda'
2021

2122
- script: |
22-
"C:\\Miniconda\\python.exe" buildscripts\\build.py --build-mode=package --python=%PYTHON_VER% --numpy=%NUMPY_VER% --conda-prefix="C:\\Miniconda"
23+
IF "%USE_NUMBA_MASTER%" == "False" (
24+
"C:\\Miniconda\\python.exe" buildscripts\\build.py --build-mode=package --python=%PYTHON_VER% --numpy=%NUMPY_VER% --conda-prefix="C:\\Miniconda"
25+
) ELSE (
26+
"C:\\Miniconda\\python.exe" buildscripts\\build.py --build-mode=package --python=%PYTHON_VER% --numpy=%NUMPY_VER% --use-numba-master --conda-prefix="C:\\Miniconda")
2327
displayName: 'Build conda and wheel packages'
28+
continueOnError: ${{ parameters.allowFailure }}
2429
2530
- script: |
26-
"C:\\Miniconda\\python.exe" buildscripts\\test.py --test-mode=conda --python=%PYTHON_VER% --numpy=%NUMPY_VER% --conda-prefix="C:\\Miniconda"
31+
IF "%USE_NUMBA_MASTER%" == "False" (
32+
"C:\\Miniconda\\python.exe" buildscripts\\test.py --test-mode=conda --python=%PYTHON_VER% --numpy=%NUMPY_VER% --conda-prefix="C:\\Miniconda"
33+
) ELSE (
34+
"C:\\Miniconda\\python.exe" buildscripts\\test.py --test-mode=conda --python=%PYTHON_VER% --numpy=%NUMPY_VER% --use-numba-master --conda-prefix="C:\\Miniconda")
2735
displayName: 'Test conda package'
36+
continueOnError: ${{ parameters.allowFailure }}

‎buildscripts/build.py‎

Lines changed: 72 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,31 @@
1+
# *****************************************************************************
2+
# Copyright (c) 2019, Intel Corporation All rights reserved.
3+
#
4+
# Redistribution and use in source and binary forms, with or without
5+
# modification, are permitted provided that the following conditions are met:
6+
#
7+
# Redistributions of source code must retain the above copyright notice,
8+
# this list of conditions and the following disclaimer.
9+
#
10+
# Redistributions in binary form must reproduce the above copyright notice,
11+
# this list of conditions and the following disclaimer in the documentation
12+
# and/or other materials provided with the distribution.
13+
#
14+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15+
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16+
# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17+
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
18+
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19+
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20+
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21+
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22+
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23+
# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
24+
# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25+
# *****************************************************************************
26+
27+
28+
129
import argparse
230
import os
331
import platform
@@ -24,6 +52,11 @@ def run_smoke_tests(sdc_src, test_env_activate):
2452
if __name__ == '__main__':
2553
sdc_src = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
2654
sdc_recipe = os.path.join(sdc_src, 'buildscripts', 'sdc-conda-recipe')
55+
numba_recipe = os.path.join(sdc_src, 'buildscripts', 'numba-conda-recipe', 'recipe')
56+
numba_output_folder = os.path.join(sdc_src, 'numba-build')
57+
numba_master_channel = f'file:/{numba_output_folder}'
58+
if platform.system() == 'Windows':
59+
numba_master_channel = f'{numba_output_folder}'
2760

2861
os.chdir(sdc_src)
2962

@@ -42,27 +75,37 @@ def run_smoke_tests(sdc_src, test_env_activate):
4275
help='Output folder for build packages, default = sdc-build')
4376
parser.add_argument('--conda-prefix', default=None, help='Conda prefix')
4477
parser.add_argument('--skip-smoke-tests', action='store_true', help='Skip smoke tests for build')
78+
parser.add_argument('--numba-channel', default='numba',
79+
help='Numba channel to build with special Numba, default=numba')
80+
parser.add_argument('--use-numba-master', action='store_true', help=f'Build with Numba from master')
81+
parser.add_argument('--channel-list', default=None, help='List of channels to use: "-c <channel> -c <channel>"')
4582

4683
args = parser.parse_args()
4784

48-
build_mode = args.build_mode
49-
python = args.python
50-
numpy = args.numpy
51-
output_folder = args.output_folder
52-
conda_prefix = os.getenv('CONDA_PREFIX', args.conda_prefix)
85+
build_mode = args.build_mode
86+
python = args.python
87+
numpy = args.numpy
88+
output_folder = args.output_folder
89+
conda_prefix = os.getenv('CONDA_PREFIX', args.conda_prefix)
5390
skip_smoke_tests = args.skip_smoke_tests
91+
channel_list = args.channel_list
92+
use_numba_master = args.use_numba_master
93+
numba_channel = numba_master_channel if use_numba_master is True else args.numba_channel
5494
assert conda_prefix is not None, 'CONDA_PREFIX is not defined; Please use --conda-prefix option or activate your conda'
5595

5696
# Init variables
57-
conda_activate = get_conda_activate_cmd(conda_prefix).replace('"', '')
58-
build_env = f'sdc-build-env-py{python}-numpy{numpy}'
59-
test_env = f'sdc-test-env-py{python}-numpy{numpy}'
60-
develop_env = f'sdc-develop-env-py{python}-numpy{numpy}'
61-
build_env_activate = get_activate_env_cmd(conda_activate, build_env)
62-
test_env_activate = get_activate_env_cmd(conda_activate, test_env)
97+
conda_activate = get_conda_activate_cmd(conda_prefix).replace('"', '')
98+
build_env = f'sdc-build-env-py{python}-numpy{numpy}'
99+
test_env = f'sdc-test-env-py{python}-numpy{numpy}'
100+
develop_env = f'sdc-develop-env-py{python}-numpy{numpy}'
101+
build_env_activate = get_activate_env_cmd(conda_activate, build_env)
102+
test_env_activate = get_activate_env_cmd(conda_activate, test_env)
63103
develop_env_activate = get_activate_env_cmd(conda_activate, develop_env)
64104

65-
conda_channels = '-c conda-forge -c numba -c intel -c defaults --override-channels'
105+
conda_channels = f'-c {numba_channel} -c conda-forge -c intel -c defaults --override-channels'
106+
numba_conda_channels = '-c conda-forge -c intel --override-channels'
107+
if channel_list:
108+
conda_channels = f'{channel_list} --override-channels'
66109

67110
conda_build_packages = ['conda-build']
68111
if platform.system() == 'Windows':
@@ -72,6 +115,18 @@ def run_smoke_tests(sdc_src, test_env_activate):
72115

73116
conda_build_packages.extend(['conda-verify', 'vc', 'vs2015_runtime', 'vs2015_win-64'])
74117

118+
# Build Numba from master
119+
if use_numba_master is True:
120+
create_conda_env(conda_activate, build_env, python, conda_build_packages)
121+
format_print('Start build Numba from master')
122+
run_command('{} && {}'.format(build_env_activate,
123+
' '.join(['conda build --no-test',
124+
f'--python {python}',
125+
f'--numpy {numpy}',
126+
f'--output-folder {numba_output_folder}',
127+
f'{numba_conda_channels} {numba_recipe}'])))
128+
format_print('NUMBA BUILD COMPETED')
129+
75130
# Get sdc build and test environment
76131
sdc_env = get_sdc_env(conda_activate, sdc_src, sdc_recipe, python, numpy, conda_channels)
77132

@@ -81,20 +136,20 @@ def run_smoke_tests(sdc_src, test_env_activate):
81136
build_cmd = '{} && {}'.format(build_env_activate,
82137
' '.join(['conda build --no-test',
83138
f'--python {python}',
84-
f'--numpy={numpy}',
139+
f'--numpy {numpy}',
85140
f'--output-folder {output_folder}',
86-
f'--prefix-length 10 {conda_channels} {sdc_recipe}']))
141+
f'{conda_channels} {sdc_recipe}']))
87142
else:
88143
create_conda_env(conda_activate, develop_env, python, sdc_env['build'], conda_channels)
89144
build_cmd = f'{develop_env_activate} && python setup.py {build_mode}'
90145

91146
# Start build
92-
format_print('Start build')
147+
format_print('START SDC BUILD')
93148
run_command(build_cmd)
94149
format_print('BUILD COMPETED')
95150

96151
# Check if smoke tests should be skipped
97-
if skip_smoke_tests == True:
152+
if skip_smoke_tests is True:
98153
format_print('Smoke tests are skipped due to "--skip-smoke-tests" option')
99154
sys.exit(0)
100155

@@ -134,4 +189,4 @@ def run_smoke_tests(sdc_src, test_env_activate):
134189
format_print('Run tests for installed sdc package')
135190
run_smoke_tests(sdc_src, develop_env_activate)
136191

137-
format_print('Smoke tests are PASSED')
192+
format_print('SMOKE TESTS ARE PASSED')

‎buildscripts/numba-conda-recipe/LICENSE.txt‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
BSD 3-clause license
2+
Copyright (c) 2015-2018, conda-forge
3+
All rights reserved.
4+
5+
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
6+
7+
1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
8+
9+
2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
10+
11+
3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
12+
13+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

0 commit comments

Comments
 (0)