cengal.build_tools.find_and_prepare_cython_modules.versions.v_0.find_and_prepare_cython_modules
Module Docstring Docstrings: http://www.python.org/dev/peps/pep-0257/
1#!/usr/bin/env python 2# coding=utf-8 3 4# Copyright © 2012-2024 ButenkoMS. All rights reserved. Contacts: <gtalk@butenkoms.space> 5# 6# Licensed under the Apache License, Version 2.0 (the "License"); 7# you may not use this file except in compliance with the License. 8# You may obtain a copy of the License at 9# 10# http://www.apache.org/licenses/LICENSE-2.0 11# 12# Unless required by applicable law or agreed to in writing, software 13# distributed under the License is distributed on an "AS IS" BASIS, 14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15# See the License for the specific language governing permissions and 16# limitations under the License. 17 18 19""" 20Module Docstring 21Docstrings: http://www.python.org/dev/peps/pep-0257/ 22""" 23 24 25__author__ = "ButenkoMS <gtalk@butenkoms.space>" 26__copyright__ = "Copyright © 2012-2024 ButenkoMS. All rights reserved. Contacts: <gtalk@butenkoms.space>" 27__credits__ = ["ButenkoMS <gtalk@butenkoms.space>", ] 28__license__ = "Apache License, Version 2.0" 29__version__ = "4.4.1" 30__maintainer__ = "ButenkoMS <gtalk@butenkoms.space>" 31__email__ = "gtalk@butenkoms.space" 32# __status__ = "Prototype" 33__status__ = "Development" 34# __status__ = "Production" 35 36 37# from distutils.dist import Distribution 38from os import environ 39 40from setuptools._distutils.dist import Distribution 41 42from cengal.file_system.path_manager import path_relative_to_src, RelativePath, get_relative_path_part, sep 43from cengal.file_system.directory_manager import current_src_dir 44from cengal.file_system.directory_manager import filtered_file_list, FilteringType, filtered_file_list_traversal, file_list_traversal, FilteringEntity, \ 45 change_current_dir, secure_current_dir 46from cengal.build_tools.prepare_cflags import prepare_cflags, concat_cflags, prepare_compile_time_env, prepare_cflags_dict, prepare_given_cflags_dict, \ 47 prepare_pyx_flags_dict 48from cengal.introspection.inspect import get_exception, entity_repr_limited_try_qualname, pifrl, pdi, class_properties_values_including_overrided 49from cengal.hardware.info.cpu import cpu_info 50from cengal.system import OS_TYPE, TEMPLATE_MODULE_NAME 51from shutil import rmtree 52from os import remove, getcwd 53from os.path import splitext, normpath, join as path_join, basename, split 54from setuptools import Extension as SetuptoolsExtension 55from Cython.Distutils import Extension as CythonExtension 56from distutils.command.build import build as build_orig 57from distutils.command.build_ext import build_ext as build_ext_orig 58from setuptools.command.sdist import sdist as sdist_orig 59import json 60import importlib 61 62from os.path import isdir, exists, isfile 63 64import setuptools 65import platform 66 67from cengal.file_system.path_manager import RelativePath, get_relative_path_part 68from cengal.file_system.directory_manager import current_src_dir 69from cengal.file_system.directory_manager import file_list_traversal, FilteringEntity 70from cengal.os.execute import prepare_params 71from setuptools.discovery import find_package_path 72import subprocess 73from cengal.build_tools.build_extensions import * 74from pprint import pprint 75from typing import List, Dict, Optional, Iterable, Callable, Sequence, Tuple, Union, Type, Set 76 77 78_PYTHON_VERSION = platform.python_version_tuple() 79BUILD_CONFIG_FILENAME: str = '__build_config.py' 80GITKEEP_FILE_NAME = '.gitkeep' 81MANIFEST_INCLUDED_FILES_FILE_NAME = 'MANIFEST.in' 82MANIFEST_INCLUDED_FILES_TEMPLATE_FILE_NAME = 'MANIFEST.in.template' 83 84 85cython_file_ext = '.pyx' 86cython_transpiled_ext = {'.c'} 87compilable_ext = {'.pyx', '.cpp', '.c++', '.cxx', '.cc'} 88codegen_files_ext = {'.c', '.cpp', '.c++', '.cxx', '.cc'} 89headers_ext = {'.h', '.hpp', '.h++', '.hh', '.hxx'} 90libs_ext = {'.lib', '.dylib', '.dll', '.so', '.exp', '.obj'} 91all_ext = cython_transpiled_ext | compilable_ext | headers_ext | libs_ext 92 93 94def find_good_packages(where: str = '.', exclude: Iterable[str] = tuple(), include: Iterable[str] = ('*',), 95 modules_to_ignore: Iterable[str] = tuple(), python_2_modules: Iterable[str] = tuple()): 96 all_packages = setuptools.find_packages(where, exclude, include) 97 good_packages = list() 98 for package in all_packages: 99 is_good = True 100 for item in modules_to_ignore: 101 if package.startswith(item): 102 is_good = False 103 break 104 105 if '3' == _PYTHON_VERSION[0]: 106 for item in python_2_modules: 107 if package.startswith(item): 108 is_good = False 109 break 110 111 if is_good: 112 good_packages.append(package) 113 114 return good_packages 115 116 117def find_package_data_filter(filtering_entity: FilteringEntity, data): 118 if FilteringEntity.dirpath == filtering_entity: 119 return True 120 elif FilteringEntity.dirname == filtering_entity: 121 return True 122 elif FilteringEntity.filename == filtering_entity: 123 dirpath, filename = data 124 return GITKEEP_FILE_NAME != filename 125 elif FilteringEntity.aggregated == filtering_entity: 126 return data 127 else: 128 raise NotImplementedError() 129 130 131def find_package_data(package_src_relative_path: str, good_packages: List[str], depth: Optional[int] = 1, root_rel: Optional[RelativePath] = None) -> Tuple[Dict[str, List[str]], List[str]]: 132 depth = depth or 0 133 depth += 1 134 root_rel = RelativePath(current_src_dir()) if root_rel is None else root_rel 135 root_path = root_rel('') 136 print(f'{root_path=}') 137 package_src_rel = RelativePath(root_rel(package_src_relative_path)) 138 package_src_path = package_src_rel('') 139 print(f'{package_src_path=}') 140 141 packages_data_dict: Dict[str, List[str]] = dict() 142 manifest_included_files: Set[str] = set() 143 for package_name in good_packages: 144 package_path = find_package_path(package_name, dict(), '') 145 package_full_path: str = root_rel(package_path) 146 package_full_path_rel: RelativePath = RelativePath(package_full_path) 147 package_data: Set[str] = set() 148 possible_data_dir: str = package_full_path_rel('data') 149 if exists(possible_data_dir) and isdir(possible_data_dir): 150 possible_data_dir_rel = RelativePath(possible_data_dir) 151 travers_result = file_list_traversal(possible_data_dir, find_package_data_filter, remove_empty_dirpaths=False) 152 for dirpath, dirnames, filenames in travers_result: 153 dirpath_rel = RelativePath(dirpath) 154 for filename in filenames: 155 file_full_path = dirpath_rel(filename) 156 package_data.add(get_relative_path_part(file_full_path, package_full_path)) 157 manifest_included_files.add(get_relative_path_part(file_full_path, root_path)) 158 159 pyx_files = filtered_file_list_traversal(package_full_path, FilteringType.including, {'.pyx'}, remove_empty_items=True) 160 for dirpath, dirnames, filenames in pyx_files: 161 dirpath_rel = RelativePath(dirpath) 162 for filename in filenames: 163 file_full_path = dirpath_rel(filename) 164 package_data.add(get_relative_path_part(file_full_path, package_full_path)) 165 manifest_included_files.add(get_relative_path_part(file_full_path, root_path)) 166 167 if package_data: 168 packages_data_dict[package_name] = list(package_data) 169 170 return packages_data_dict, list(manifest_included_files) 171 172 173def generate_manifest_included_files(manifest_included_files: List[str], root_rel: RelativePath): 174 manifest_included_files_template_path: str = root_rel(MANIFEST_INCLUDED_FILES_TEMPLATE_FILE_NAME) 175 manifest_included_files_template_contents: str = str() 176 if exists(manifest_included_files_template_path) and isfile(manifest_included_files_template_path): 177 with open(manifest_included_files_template_path, 'r') as f: 178 manifest_included_files_template_contents = f.read() 179 180 manifest_included_files_path: str = root_rel(MANIFEST_INCLUDED_FILES_FILE_NAME) 181 with open(manifest_included_files_path, 'w') as f: 182 if manifest_included_files_template_contents: 183 f.write(manifest_included_files_template_contents + '\n') 184 185 for file in manifest_included_files: 186 f.write(f'include {file}\n') 187 188 189def get_file_name_without_extension(path): 190 file_name = basename(path) # Get the base name from the path 191 file_name_without_extension = splitext(file_name)[0] # Split the file name and extension, and take only the file name part 192 return file_name_without_extension 193 194 195def remove_header_files(ext_modules: Sequence[CythonExtension]) -> Sequence[CythonExtension]: 196 for ext_module in ext_modules: 197 if isinstance(ext_module, CythonExtension): 198 new_sources = list() 199 for source in ext_module.sources: 200 _, file_extension = splitext(source) 201 if file_extension not in headers_ext: 202 new_sources.append(source) 203 204 ext_module.sources = new_sources 205 206 return ext_modules 207 208 209def extend_file_names_to_root_relative_paths(root_path: str, dir_path_obj: RelativePath, file_names_or_path: Sequence[str]) -> Sequence[str]: 210 result = list() 211 for file_name_or_path in file_names_or_path: 212 if not file_name_or_path: 213 continue 214 215 owner_path, file_name = split(file_name_or_path) 216 file_name = file_name.strip() 217 if owner_path: 218 result.append(file_name_or_path) 219 else: 220 result.append(path_join('.', get_relative_path_part(dir_path_obj(file_name), root_path))) 221 222 return result 223 224 225def process_macros(ext_modules: Sequence[CythonExtension]) -> Sequence[CythonExtension]: 226 for ext_module in ext_modules: 227 if isinstance(ext_module, CythonExtension): 228 if hasattr(ext_module, 'compile_time_env'): 229 compile_time_env = ext_module.compile_time_env 230 else: 231 compile_time_env = dict() 232 233 if hasattr(ext_module, 'cython_compile_time_env'): 234 cython_compile_time_env = ext_module.cython_compile_time_env or dict() 235 compile_time_env.update(cython_compile_time_env) 236 ext_module.cython_compile_time_env = dict() 237 238 ext_module.compile_time_env = compile_time_env 239 240 if hasattr(ext_module, 'define_macros'): 241 define_macros = ext_module.define_macros 242 else: 243 define_macros = list() 244 245 if isinstance(define_macros, dict): 246 define_macros = list(define_macros.items()) 247 248 new_define_macros = list() 249 for macros_name, macros_value in define_macros: 250 macros_value = str(macros_value) 251 macros_value = macros_value.strip() 252 if not macros_value: 253 macros_value = None 254 255 new_define_macros.append((macros_name, macros_value)) 256 257 ext_module.define_macros = define_macros 258 259 return ext_modules 260 261 262def find_and_prepare_cython_modules(package_src_relative_path: str, additional_cflags: Dict[str, Tuple[bool, Union[str, int]]], depth = 1, root_rel: Optional[RelativePath] = None): 263 with secure_current_dir(): 264 depth = depth or 0 265 depth += 1 266 root_rel = RelativePath(current_src_dir(depth)) if root_rel is None else root_rel 267 root_path = root_rel('') 268 package_src_rel = RelativePath(root_rel(package_src_relative_path)) 269 package_src_path = package_src_rel('') 270 271 travers_result = filtered_file_list_traversal(package_src_path, FilteringType.off, None, remove_empty_items=True, use_spinner=False) 272 for dir_path, dirs, files in travers_result: 273 if '__pycache__' in dirs: 274 pycache_path = RelativePath(dir_path)('__pycache__') 275 try: 276 rmtree(pycache_path) 277 except OSError as e: 278 print(get_exception()) 279 280 281 def filter(filtering_entity: FilteringEntity, data): 282 if FilteringEntity.dirpath == filtering_entity: 283 # Ignore package_src/_template_module 284 dirpath, dirnames, filenames = data 285 rel_path = get_relative_path_part(dirpath, package_src_path) 286 rel_path = rel_path.strip() 287 if rel_path: 288 rel_path_parts = rel_path.split(sep) 289 if rel_path_parts and TEMPLATE_MODULE_NAME == rel_path_parts[0]: 290 return False 291 292 return True 293 elif FilteringEntity.dirname == filtering_entity: 294 return True 295 elif FilteringEntity.filename == filtering_entity: 296 dirpath, filename = data 297 if BUILD_CONFIG_FILENAME == filename: 298 return True 299 300 file_name, file_extension = splitext(filename) 301 if file_extension in all_ext: 302 return True 303 304 return False 305 elif FilteringEntity.aggregated == filtering_entity: 306 return data 307 else: 308 raise NotImplementedError() 309 310 travers_result = file_list_traversal(package_src_path, filter, remove_empty_dirpaths=True) 311 # travers_result = filtered_file_list_traversal(package_src_path, FilteringType.including, {'.pyx', '.c', '.lib', '.dll', '.so'}, remove_empty_items=True, use_spinner=False) 312 313 result = list() 314 for dir_path, dirs, files in travers_result: 315 # Ignore package_src/_template_module 316 rel_path = get_relative_path_part(dir_path, package_src_path) 317 rel_path = rel_path.strip() 318 if rel_path: 319 rel_path_parts = rel_path.split(sep) 320 if rel_path_parts and TEMPLATE_MODULE_NAME == rel_path_parts[0]: 321 continue 322 323 dir_path_obj = RelativePath(dir_path) 324 extensions = dict() 325 for file in files: 326 filename, file_extension = splitext(file) 327 if file_extension not in extensions: 328 extensions[file_extension] = set() 329 330 extensions[file_extension].add(file) 331 332 is_exctension: bool = False 333 build_config: dict = None 334 if BUILD_CONFIG_FILENAME in files: 335 build_config_module_name, _ = splitext(BUILD_CONFIG_FILENAME) 336 build_config_full_path: str = dir_path_obj(build_config_module_name) 337 name = get_relative_path_part(build_config_full_path, root_rel('')) 338 name_parts = name.split(sep) 339 module_full_name = '.'.join([name_part for name_part in name_parts if name_part]) 340 build_config_module = importlib.import_module(module_full_name) 341 build_config = build_config_module.build_config() 342 is_exctension = True 343 344 sub_result = list() 345 if (cython_file_ext in extensions) or (build_config is not None): 346 for file_extension, files in extensions.items(): 347 for file in files: 348 if file_extension.lower() in (compilable_ext | headers_ext): 349 is_exctension = True 350 sub_result.append(path_join('.', get_relative_path_part(dir_path_obj(file), root_path))) 351 352 if file_extension.lower() in codegen_files_ext: 353 filename, _ = splitext(file) 354 if filename.endswith('__cython') or filename.endswith('__compiled') or filename.endswith('__python'): 355 try: 356 remove(dir_path_obj(file)) 357 except OSError as e: 358 print(get_exception()) 359 else: 360 is_exctension = True 361 sub_result.append(path_join('.', get_relative_path_part(dir_path_obj(file), root_path))) 362 363 if file_extension.lower() in libs_ext: 364 try: 365 remove(dir_path_obj(file)) 366 except OSError as e: 367 print(get_exception()) 368 369 if is_exctension: 370 name = get_relative_path_part(dir_path_obj(''), root_rel('')) 371 name_parts = name.split(sep) 372 if build_config is None: 373 if 1 == len(sub_result) and sub_result[0].endswith(cython_file_ext): 374 name_parts.append(get_file_name_without_extension(sub_result[0])) 375 else: 376 name_parts.append('cython_module') 377 378 name = '.'.join([name_part for name_part in name_parts if name_part]) 379 extension: CythonExtension = CythonExtension( 380 name, 381 sources=list(set(sub_result)), 382 language="c", 383 # cython_compile_time_env=prepare_cflags_dict(additional_cflags), 384 define_macros=prepare_cflags_dict(additional_cflags), 385 ) 386 else: 387 if isinstance(build_config, (dict, tuple)): 388 extension_type = CythonExtension 389 if isinstance(build_config, tuple): 390 extension_type, build_config = build_config 391 extension_type = extension_type if extension_type is not None else CythonExtension 392 393 if 'Windows' == OS_TYPE: 394 # CythonExtension :raises setuptools.errors.PlatformError: if 'runtime_library_dirs' is specified on Windows. (since v63) 395 build_config.pop('runtime_library_dirs', None) 396 397 name_parts.append(build_config['name']) 398 name = '.'.join([name_part for name_part in name_parts if name_part]) 399 build_config.pop('name', None) 400 401 cython_compile_time_env=prepare_cflags_dict(additional_cflags) 402 if 'cython_compile_time_env' in build_config: 403 cython_compile_time_env.update(build_config['cython_compile_time_env']) 404 build_config.pop('cython_compile_time_env', None) 405 406 define_macros=prepare_cflags_dict(additional_cflags) 407 if 'define_macros' in build_config: 408 define_macros.update(build_config['define_macros']) 409 build_config.pop('define_macros', None) 410 411 sources = extend_file_names_to_root_relative_paths(root_path, dir_path_obj, build_config.pop('sources', list())) 412 sources.extend(sub_result) 413 414 extension: CythonExtension = extension_type( 415 name, 416 sources=list(set(sources)), 417 # cython_compile_time_env=cython_compile_time_env, 418 define_macros=define_macros, 419 **build_config 420 ) 421 elif isinstance(build_config, CengalBuildExtension): 422 build_config.path = dir_path_obj(BUILD_CONFIG_FILENAME) 423 build_config.module_import_str = module_full_name 424 build_config.files.extend(sub_result) 425 extension = build_config 426 else: 427 raise RuntimeError(f'Unknown build_config type: {type(build_config)}') 428 429 result.append(extension) 430 else: 431 result.extend(sub_result) 432 433 return result 434 435 436class BuildConfig: 437 def __init__(self) -> None: 438 self.package_build_is_in_debug_mode: str = str() 439 self.package_build_is_in_sdist_mode: str = str() 440 self.additional_pyx_flags: Dict[str, Tuple[bool, Union[str, int]]] = dict() 441 self.find_package_data: Callable = None 442 self.root_rel: RelativePath = None 443 444 445class sdist(sdist_orig): 446 def __init__(self, dist: Distribution, build_config: BuildConfig) -> None: 447 self.build_config: BuildConfig = build_config 448 super().__init__(dist) 449 450 def run(self): 451 # print('<<< sdist command is currently being run >>>') 452 with secure_current_dir(): 453 if self.build_config.package_build_is_in_debug_mode in environ: 454 import debugpy 455 debugpy.breakpoint() 456 457 # print("sdist command is currently being run") 458 environ[self.build_config.package_build_is_in_sdist_mode] = 'True' 459 packages_data_dict, manifest_included_files = self.build_config.find_package_data() 460 # from pprint import pprint 461 # print() 462 # print('<<< packages_data_dict: >>>') 463 # pprint(packages_data_dict) 464 # print() 465 # print('<<< manifest_included_files: >>>') 466 # pprint(manifest_included_files) 467 # print() 468 generate_manifest_included_files(manifest_included_files, self.build_config.root_rel) 469 470 super().run() 471 472 473class build(build_orig): 474 def __init__(self, dist: Distribution, build_config: BuildConfig) -> None: 475 self.build_config: BuildConfig = build_config 476 super().__init__(dist) 477 478 def finalize_options(self): 479 # print('<<< build.finalize_options is currently being run >>>') 480 try: 481 with secure_current_dir(): 482 setuptools_extensions = [ext for ext in self.distribution.ext_modules if isinstance(ext, SetuptoolsExtension)] 483 cython_extensions = [ext for ext in self.distribution.ext_modules if isinstance(ext, CythonExtension)] 484 cengal_extensions = (ext for ext in self.distribution.ext_modules if (isinstance(ext, CengalBuildExtension) and (not ext.store_as_data))) 485 cengal_result_extensions = [ext() for ext in cengal_extensions] 486 setuptools_extensions.extend([ext for ext in cengal_result_extensions if isinstance(ext, SetuptoolsExtension)]) 487 cython_extensions.extend([ext for ext in cengal_result_extensions if isinstance(ext, CythonExtension)]) 488 cengal_extensions_store_as_data = (ext for ext in self.distribution.ext_modules if (isinstance(ext, CengalBuildExtension) and ext.store_as_data)) 489 490 if self.build_config.package_build_is_in_debug_mode in environ: 491 import debugpy 492 debugpy.breakpoint() 493 494 super().finalize_options() 495 496 with secure_current_dir(): 497 package_data = self.distribution.package_data or dict() 498 499 if self.build_config.package_build_is_in_sdist_mode in environ: 500 for ext in cengal_extensions_store_as_data: 501 files = ext.sdist() 502 if files is None: 503 continue 504 505 if ext.package not in package_data: 506 package_data[ext.package] = list() 507 508 package_data[ext.package].extend(files) 509 510 result_ext_modules = list() 511 result_ext_modules.extend(cython_extensions) 512 result_ext_modules.extend(setuptools_extensions) 513 self.distribution.package_data = package_data 514 self.distribution.ext_modules = result_ext_modules 515 # from pprint import pprint 516 # print() 517 # print('<<< self.distribution.package_data: >>>') 518 # pprint(self.distribution.package_data) 519 # print() 520 # print('<<< self.distribution.ext_modules: >>>') 521 # pprint(self.distribution.ext_modules) 522 # print() 523 else: 524 for ext in cengal_extensions_store_as_data: 525 files = ext() 526 if files is None: 527 continue 528 529 if ext.package not in package_data: 530 package_data[ext.package] = list() 531 532 package_data[ext.package].extend(files) 533 534 from Cython.Build import cythonize 535 result_ext_modules = list() 536 try: 537 cwd_before_cythonize = getcwd() 538 pyx_flags_dict = prepare_pyx_flags_dict(self.build_config.additional_pyx_flags) 539 result_ext_modules.extend(remove_header_files( 540 cythonize(process_macros(cython_extensions), 541 compiler_directives={'language_level': '3'}, 542 compile_time_env = pyx_flags_dict, 543 ))) 544 except: 545 print(f'DEBUG: {cwd_before_cythonize=} | {getcwd()=}') 546 for extension in cython_extensions: 547 pprint(class_properties_values_including_overrided(extension)) 548 549 raise 550 551 result_ext_modules.extend(setuptools_extensions) 552 for ext in result_ext_modules: 553 if not hasattr(ext, 'extra_compile_args'): 554 ext.extra_compile_args = list() 555 556 if ext.extra_compile_args is None: 557 ext.extra_compile_args = list() 558 559 if 'Darwin' == OS_TYPE: 560 if cpu_info().is_x86: 561 ext.extra_compile_args += ['-arch', 'x86_64'] 562 elif cpu_info().is_arm: 563 ext.extra_compile_args += ['-arch', 'arm64'] 564 565 self.distribution.package_data = package_data 566 self.distribution.ext_modules = result_ext_modules 567 568 if self.build_config.package_build_is_in_debug_mode in environ: 569 debugpy.breakpoint() 570 print() 571 finally: 572 # print('<<< build.finalize_options is done >>>') 573 pass 574 575 576class build_ext(build_ext_orig): 577 def __init__(self, dist: Distribution, build_config: BuildConfig) -> None: 578 self.build_config: BuildConfig = build_config 579 super().__init__(dist) 580 581 def finalize_options(self): 582 with secure_current_dir(): 583 setuptools_extensions = [ext for ext in self.distribution.ext_modules if isinstance(ext, SetuptoolsExtension)] 584 cython_extensions = [ext for ext in self.distribution.ext_modules if isinstance(ext, CythonExtension)] 585 cengal_extensions = (ext for ext in self.distribution.ext_modules if (isinstance(ext, CengalBuildExtension) and (not ext.store_as_data))) 586 cengal_result_extensions = [ext() for ext in cengal_extensions] 587 setuptools_extensions.extend([ext for ext in cengal_result_extensions if isinstance(ext, SetuptoolsExtension)]) 588 cython_extensions.extend([ext for ext in cengal_result_extensions if isinstance(ext, CythonExtension)]) 589 cengal_extensions_store_as_data = (ext for ext in self.distribution.ext_modules if (isinstance(ext, CengalBuildExtension) and ext.store_as_data)) 590 591 if self.build_config.package_build_is_in_debug_mode in environ: 592 import debugpy 593 debugpy.breakpoint() 594 595 super().finalize_options() 596 597 with secure_current_dir(): 598 package_data = self.distribution.package_data or dict() 599 600 if self.build_config.package_build_is_in_sdist_mode not in environ: 601 for ext in cengal_extensions_store_as_data: 602 files = ext() 603 if files is None: 604 continue 605 606 if ext.package not in package_data: 607 package_data[ext.package] = list() 608 609 package_data[ext.package].extend(files) 610 611 from Cython.Build import cythonize 612 result_ext_modules = list() 613 try: 614 cwd_before_cythonize = getcwd() 615 pyx_flags_dict = prepare_pyx_flags_dict(self.build_config.additional_pyx_flags) 616 result_ext_modules.extend(remove_header_files( 617 cythonize(process_macros(cython_extensions), 618 compiler_directives={'language_level': '3'}, 619 compile_time_env = pyx_flags_dict, 620 ))) 621 except: 622 print(f'DEBUG: {cwd_before_cythonize=} | {getcwd()=}') 623 for extension in cython_extensions: 624 pprint(class_properties_values_including_overrided(extension)) 625 626 raise 627 628 result_ext_modules.extend(setuptools_extensions) 629 for ext in result_ext_modules: 630 if not hasattr(ext, 'extra_compile_args'): 631 ext.extra_compile_args = list() 632 633 if ext.extra_compile_args is None: 634 ext.extra_compile_args = list() 635 636 if 'Darwin' == OS_TYPE: 637 if cpu_info().is_x86: 638 ext.extra_compile_args += ['-arch', 'x86_64'] 639 elif cpu_info().is_arm: 640 ext.extra_compile_args += ['-arch', 'arm64'] 641 642 self.distribution.package_data = package_data 643 self.distribution.ext_modules = result_ext_modules 644 645 if self.build_config.package_build_is_in_debug_mode in environ: 646 debugpy.breakpoint() 647 print()
BUILD_CONFIG_FILENAME: str =
'__build_config.py'
GITKEEP_FILE_NAME =
'.gitkeep'
MANIFEST_INCLUDED_FILES_FILE_NAME =
'MANIFEST.in'
MANIFEST_INCLUDED_FILES_TEMPLATE_FILE_NAME =
'MANIFEST.in.template'
cython_file_ext =
'.pyx'
cython_transpiled_ext =
{'.c'}
compilable_ext =
{'.c++', '.cxx', '.pyx', '.cpp', '.cc'}
codegen_files_ext =
{'.c++', '.c', '.cxx', '.cpp', '.cc'}
headers_ext =
{'.hpp', '.h', '.hxx', '.hh', '.h++'}
libs_ext =
{'.exp', '.dylib', '.lib', '.so', '.dll', '.obj'}
all_ext =
{'.c++', '.c', '.h++', '.hpp', '.exp', '.dylib', '.h', '.cxx', '.lib', '.so', '.pyx', '.hh', '.dll', '.hxx', '.cpp', '.obj', '.cc'}
def
find_good_packages( where: str = '.', exclude: Iterable[str] = (), include: Iterable[str] = ('*',), modules_to_ignore: Iterable[str] = (), python_2_modules: Iterable[str] = ()):
95def find_good_packages(where: str = '.', exclude: Iterable[str] = tuple(), include: Iterable[str] = ('*',), 96 modules_to_ignore: Iterable[str] = tuple(), python_2_modules: Iterable[str] = tuple()): 97 all_packages = setuptools.find_packages(where, exclude, include) 98 good_packages = list() 99 for package in all_packages: 100 is_good = True 101 for item in modules_to_ignore: 102 if package.startswith(item): 103 is_good = False 104 break 105 106 if '3' == _PYTHON_VERSION[0]: 107 for item in python_2_modules: 108 if package.startswith(item): 109 is_good = False 110 break 111 112 if is_good: 113 good_packages.append(package) 114 115 return good_packages
def
find_package_data_filter( filtering_entity: cengal.file_system.directory_manager.versions.v_0.directory_manager.FilteringEntity, data):
118def find_package_data_filter(filtering_entity: FilteringEntity, data): 119 if FilteringEntity.dirpath == filtering_entity: 120 return True 121 elif FilteringEntity.dirname == filtering_entity: 122 return True 123 elif FilteringEntity.filename == filtering_entity: 124 dirpath, filename = data 125 return GITKEEP_FILE_NAME != filename 126 elif FilteringEntity.aggregated == filtering_entity: 127 return data 128 else: 129 raise NotImplementedError()
def
find_package_data( package_src_relative_path: str, good_packages: List[str], depth: Union[int, NoneType] = 1, root_rel: Union[cengal.file_system.path_manager.versions.v_0.path_manager.RelativePath, NoneType] = None) -> Tuple[Dict[str, List[str]], List[str]]:
132def find_package_data(package_src_relative_path: str, good_packages: List[str], depth: Optional[int] = 1, root_rel: Optional[RelativePath] = None) -> Tuple[Dict[str, List[str]], List[str]]: 133 depth = depth or 0 134 depth += 1 135 root_rel = RelativePath(current_src_dir()) if root_rel is None else root_rel 136 root_path = root_rel('') 137 print(f'{root_path=}') 138 package_src_rel = RelativePath(root_rel(package_src_relative_path)) 139 package_src_path = package_src_rel('') 140 print(f'{package_src_path=}') 141 142 packages_data_dict: Dict[str, List[str]] = dict() 143 manifest_included_files: Set[str] = set() 144 for package_name in good_packages: 145 package_path = find_package_path(package_name, dict(), '') 146 package_full_path: str = root_rel(package_path) 147 package_full_path_rel: RelativePath = RelativePath(package_full_path) 148 package_data: Set[str] = set() 149 possible_data_dir: str = package_full_path_rel('data') 150 if exists(possible_data_dir) and isdir(possible_data_dir): 151 possible_data_dir_rel = RelativePath(possible_data_dir) 152 travers_result = file_list_traversal(possible_data_dir, find_package_data_filter, remove_empty_dirpaths=False) 153 for dirpath, dirnames, filenames in travers_result: 154 dirpath_rel = RelativePath(dirpath) 155 for filename in filenames: 156 file_full_path = dirpath_rel(filename) 157 package_data.add(get_relative_path_part(file_full_path, package_full_path)) 158 manifest_included_files.add(get_relative_path_part(file_full_path, root_path)) 159 160 pyx_files = filtered_file_list_traversal(package_full_path, FilteringType.including, {'.pyx'}, remove_empty_items=True) 161 for dirpath, dirnames, filenames in pyx_files: 162 dirpath_rel = RelativePath(dirpath) 163 for filename in filenames: 164 file_full_path = dirpath_rel(filename) 165 package_data.add(get_relative_path_part(file_full_path, package_full_path)) 166 manifest_included_files.add(get_relative_path_part(file_full_path, root_path)) 167 168 if package_data: 169 packages_data_dict[package_name] = list(package_data) 170 171 return packages_data_dict, list(manifest_included_files)
def
generate_manifest_included_files( manifest_included_files: List[str], root_rel: cengal.file_system.path_manager.versions.v_0.path_manager.RelativePath):
174def generate_manifest_included_files(manifest_included_files: List[str], root_rel: RelativePath): 175 manifest_included_files_template_path: str = root_rel(MANIFEST_INCLUDED_FILES_TEMPLATE_FILE_NAME) 176 manifest_included_files_template_contents: str = str() 177 if exists(manifest_included_files_template_path) and isfile(manifest_included_files_template_path): 178 with open(manifest_included_files_template_path, 'r') as f: 179 manifest_included_files_template_contents = f.read() 180 181 manifest_included_files_path: str = root_rel(MANIFEST_INCLUDED_FILES_FILE_NAME) 182 with open(manifest_included_files_path, 'w') as f: 183 if manifest_included_files_template_contents: 184 f.write(manifest_included_files_template_contents + '\n') 185 186 for file in manifest_included_files: 187 f.write(f'include {file}\n')
def
get_file_name_without_extension(path):
def
remove_header_files( ext_modules: Sequence[Cython.Distutils.extension.Extension]) -> Sequence[Cython.Distutils.extension.Extension]:
196def remove_header_files(ext_modules: Sequence[CythonExtension]) -> Sequence[CythonExtension]: 197 for ext_module in ext_modules: 198 if isinstance(ext_module, CythonExtension): 199 new_sources = list() 200 for source in ext_module.sources: 201 _, file_extension = splitext(source) 202 if file_extension not in headers_ext: 203 new_sources.append(source) 204 205 ext_module.sources = new_sources 206 207 return ext_modules
def
extend_file_names_to_root_relative_paths( root_path: str, dir_path_obj: cengal.file_system.path_manager.versions.v_0.path_manager.RelativePath, file_names_or_path: Sequence[str]) -> Sequence[str]:
210def extend_file_names_to_root_relative_paths(root_path: str, dir_path_obj: RelativePath, file_names_or_path: Sequence[str]) -> Sequence[str]: 211 result = list() 212 for file_name_or_path in file_names_or_path: 213 if not file_name_or_path: 214 continue 215 216 owner_path, file_name = split(file_name_or_path) 217 file_name = file_name.strip() 218 if owner_path: 219 result.append(file_name_or_path) 220 else: 221 result.append(path_join('.', get_relative_path_part(dir_path_obj(file_name), root_path))) 222 223 return result
def
process_macros( ext_modules: Sequence[Cython.Distutils.extension.Extension]) -> Sequence[Cython.Distutils.extension.Extension]:
226def process_macros(ext_modules: Sequence[CythonExtension]) -> Sequence[CythonExtension]: 227 for ext_module in ext_modules: 228 if isinstance(ext_module, CythonExtension): 229 if hasattr(ext_module, 'compile_time_env'): 230 compile_time_env = ext_module.compile_time_env 231 else: 232 compile_time_env = dict() 233 234 if hasattr(ext_module, 'cython_compile_time_env'): 235 cython_compile_time_env = ext_module.cython_compile_time_env or dict() 236 compile_time_env.update(cython_compile_time_env) 237 ext_module.cython_compile_time_env = dict() 238 239 ext_module.compile_time_env = compile_time_env 240 241 if hasattr(ext_module, 'define_macros'): 242 define_macros = ext_module.define_macros 243 else: 244 define_macros = list() 245 246 if isinstance(define_macros, dict): 247 define_macros = list(define_macros.items()) 248 249 new_define_macros = list() 250 for macros_name, macros_value in define_macros: 251 macros_value = str(macros_value) 252 macros_value = macros_value.strip() 253 if not macros_value: 254 macros_value = None 255 256 new_define_macros.append((macros_name, macros_value)) 257 258 ext_module.define_macros = define_macros 259 260 return ext_modules
def
find_and_prepare_cython_modules( package_src_relative_path: str, additional_cflags: Dict[str, Tuple[bool, Union[str, int]]], depth=1, root_rel: Union[cengal.file_system.path_manager.versions.v_0.path_manager.RelativePath, NoneType] = None):
263def find_and_prepare_cython_modules(package_src_relative_path: str, additional_cflags: Dict[str, Tuple[bool, Union[str, int]]], depth = 1, root_rel: Optional[RelativePath] = None): 264 with secure_current_dir(): 265 depth = depth or 0 266 depth += 1 267 root_rel = RelativePath(current_src_dir(depth)) if root_rel is None else root_rel 268 root_path = root_rel('') 269 package_src_rel = RelativePath(root_rel(package_src_relative_path)) 270 package_src_path = package_src_rel('') 271 272 travers_result = filtered_file_list_traversal(package_src_path, FilteringType.off, None, remove_empty_items=True, use_spinner=False) 273 for dir_path, dirs, files in travers_result: 274 if '__pycache__' in dirs: 275 pycache_path = RelativePath(dir_path)('__pycache__') 276 try: 277 rmtree(pycache_path) 278 except OSError as e: 279 print(get_exception()) 280 281 282 def filter(filtering_entity: FilteringEntity, data): 283 if FilteringEntity.dirpath == filtering_entity: 284 # Ignore package_src/_template_module 285 dirpath, dirnames, filenames = data 286 rel_path = get_relative_path_part(dirpath, package_src_path) 287 rel_path = rel_path.strip() 288 if rel_path: 289 rel_path_parts = rel_path.split(sep) 290 if rel_path_parts and TEMPLATE_MODULE_NAME == rel_path_parts[0]: 291 return False 292 293 return True 294 elif FilteringEntity.dirname == filtering_entity: 295 return True 296 elif FilteringEntity.filename == filtering_entity: 297 dirpath, filename = data 298 if BUILD_CONFIG_FILENAME == filename: 299 return True 300 301 file_name, file_extension = splitext(filename) 302 if file_extension in all_ext: 303 return True 304 305 return False 306 elif FilteringEntity.aggregated == filtering_entity: 307 return data 308 else: 309 raise NotImplementedError() 310 311 travers_result = file_list_traversal(package_src_path, filter, remove_empty_dirpaths=True) 312 # travers_result = filtered_file_list_traversal(package_src_path, FilteringType.including, {'.pyx', '.c', '.lib', '.dll', '.so'}, remove_empty_items=True, use_spinner=False) 313 314 result = list() 315 for dir_path, dirs, files in travers_result: 316 # Ignore package_src/_template_module 317 rel_path = get_relative_path_part(dir_path, package_src_path) 318 rel_path = rel_path.strip() 319 if rel_path: 320 rel_path_parts = rel_path.split(sep) 321 if rel_path_parts and TEMPLATE_MODULE_NAME == rel_path_parts[0]: 322 continue 323 324 dir_path_obj = RelativePath(dir_path) 325 extensions = dict() 326 for file in files: 327 filename, file_extension = splitext(file) 328 if file_extension not in extensions: 329 extensions[file_extension] = set() 330 331 extensions[file_extension].add(file) 332 333 is_exctension: bool = False 334 build_config: dict = None 335 if BUILD_CONFIG_FILENAME in files: 336 build_config_module_name, _ = splitext(BUILD_CONFIG_FILENAME) 337 build_config_full_path: str = dir_path_obj(build_config_module_name) 338 name = get_relative_path_part(build_config_full_path, root_rel('')) 339 name_parts = name.split(sep) 340 module_full_name = '.'.join([name_part for name_part in name_parts if name_part]) 341 build_config_module = importlib.import_module(module_full_name) 342 build_config = build_config_module.build_config() 343 is_exctension = True 344 345 sub_result = list() 346 if (cython_file_ext in extensions) or (build_config is not None): 347 for file_extension, files in extensions.items(): 348 for file in files: 349 if file_extension.lower() in (compilable_ext | headers_ext): 350 is_exctension = True 351 sub_result.append(path_join('.', get_relative_path_part(dir_path_obj(file), root_path))) 352 353 if file_extension.lower() in codegen_files_ext: 354 filename, _ = splitext(file) 355 if filename.endswith('__cython') or filename.endswith('__compiled') or filename.endswith('__python'): 356 try: 357 remove(dir_path_obj(file)) 358 except OSError as e: 359 print(get_exception()) 360 else: 361 is_exctension = True 362 sub_result.append(path_join('.', get_relative_path_part(dir_path_obj(file), root_path))) 363 364 if file_extension.lower() in libs_ext: 365 try: 366 remove(dir_path_obj(file)) 367 except OSError as e: 368 print(get_exception()) 369 370 if is_exctension: 371 name = get_relative_path_part(dir_path_obj(''), root_rel('')) 372 name_parts = name.split(sep) 373 if build_config is None: 374 if 1 == len(sub_result) and sub_result[0].endswith(cython_file_ext): 375 name_parts.append(get_file_name_without_extension(sub_result[0])) 376 else: 377 name_parts.append('cython_module') 378 379 name = '.'.join([name_part for name_part in name_parts if name_part]) 380 extension: CythonExtension = CythonExtension( 381 name, 382 sources=list(set(sub_result)), 383 language="c", 384 # cython_compile_time_env=prepare_cflags_dict(additional_cflags), 385 define_macros=prepare_cflags_dict(additional_cflags), 386 ) 387 else: 388 if isinstance(build_config, (dict, tuple)): 389 extension_type = CythonExtension 390 if isinstance(build_config, tuple): 391 extension_type, build_config = build_config 392 extension_type = extension_type if extension_type is not None else CythonExtension 393 394 if 'Windows' == OS_TYPE: 395 # CythonExtension :raises setuptools.errors.PlatformError: if 'runtime_library_dirs' is specified on Windows. (since v63) 396 build_config.pop('runtime_library_dirs', None) 397 398 name_parts.append(build_config['name']) 399 name = '.'.join([name_part for name_part in name_parts if name_part]) 400 build_config.pop('name', None) 401 402 cython_compile_time_env=prepare_cflags_dict(additional_cflags) 403 if 'cython_compile_time_env' in build_config: 404 cython_compile_time_env.update(build_config['cython_compile_time_env']) 405 build_config.pop('cython_compile_time_env', None) 406 407 define_macros=prepare_cflags_dict(additional_cflags) 408 if 'define_macros' in build_config: 409 define_macros.update(build_config['define_macros']) 410 build_config.pop('define_macros', None) 411 412 sources = extend_file_names_to_root_relative_paths(root_path, dir_path_obj, build_config.pop('sources', list())) 413 sources.extend(sub_result) 414 415 extension: CythonExtension = extension_type( 416 name, 417 sources=list(set(sources)), 418 # cython_compile_time_env=cython_compile_time_env, 419 define_macros=define_macros, 420 **build_config 421 ) 422 elif isinstance(build_config, CengalBuildExtension): 423 build_config.path = dir_path_obj(BUILD_CONFIG_FILENAME) 424 build_config.module_import_str = module_full_name 425 build_config.files.extend(sub_result) 426 extension = build_config 427 else: 428 raise RuntimeError(f'Unknown build_config type: {type(build_config)}') 429 430 result.append(extension) 431 else: 432 result.extend(sub_result) 433 434 return result
class
BuildConfig:
437class BuildConfig: 438 def __init__(self) -> None: 439 self.package_build_is_in_debug_mode: str = str() 440 self.package_build_is_in_sdist_mode: str = str() 441 self.additional_pyx_flags: Dict[str, Tuple[bool, Union[str, int]]] = dict() 442 self.find_package_data: Callable = None 443 self.root_rel: RelativePath = None
class
sdist(setuptools.command.sdist.sdist):
446class sdist(sdist_orig): 447 def __init__(self, dist: Distribution, build_config: BuildConfig) -> None: 448 self.build_config: BuildConfig = build_config 449 super().__init__(dist) 450 451 def run(self): 452 # print('<<< sdist command is currently being run >>>') 453 with secure_current_dir(): 454 if self.build_config.package_build_is_in_debug_mode in environ: 455 import debugpy 456 debugpy.breakpoint() 457 458 # print("sdist command is currently being run") 459 environ[self.build_config.package_build_is_in_sdist_mode] = 'True' 460 packages_data_dict, manifest_included_files = self.build_config.find_package_data() 461 # from pprint import pprint 462 # print() 463 # print('<<< packages_data_dict: >>>') 464 # pprint(packages_data_dict) 465 # print() 466 # print('<<< manifest_included_files: >>>') 467 # pprint(manifest_included_files) 468 # print() 469 generate_manifest_included_files(manifest_included_files, self.build_config.root_rel) 470 471 super().run()
Smart sdist that finds anything supported by revision control
sdist( dist: setuptools._distutils.dist.Distribution, build_config: BuildConfig)
447 def __init__(self, dist: Distribution, build_config: BuildConfig) -> None: 448 self.build_config: BuildConfig = build_config 449 super().__init__(dist)
Construct the command for dist, updating vars(self) with any keyword parameters.
build_config: BuildConfig
def
run(self):
451 def run(self): 452 # print('<<< sdist command is currently being run >>>') 453 with secure_current_dir(): 454 if self.build_config.package_build_is_in_debug_mode in environ: 455 import debugpy 456 debugpy.breakpoint() 457 458 # print("sdist command is currently being run") 459 environ[self.build_config.package_build_is_in_sdist_mode] = 'True' 460 packages_data_dict, manifest_included_files = self.build_config.find_package_data() 461 # from pprint import pprint 462 # print() 463 # print('<<< packages_data_dict: >>>') 464 # pprint(packages_data_dict) 465 # print() 466 # print('<<< manifest_included_files: >>>') 467 # pprint(manifest_included_files) 468 # print() 469 generate_manifest_included_files(manifest_included_files, self.build_config.root_rel) 470 471 super().run()
A command's raison d'etre: carry out the action it exists to perform, controlled by the options initialized in 'initialize_options()', customized by other commands, the setup script, the command-line, and config files, and finalized in 'finalize_options()'. All terminal output and filesystem interaction should be done by 'run()'.
This method must be implemented by all command classes.
Inherited Members
- setuptools.command.sdist.sdist
- user_options
- negative_opt
- README_EXTENSIONS
- READMES
- initialize_options
- make_distribution
- add_defaults
- check_readme
- make_release_tree
- read_manifest
- distutils.command.sdist.sdist
- description
- checking_metadata
- boolean_options
- help_options
- sub_commands
- finalize_options
- check_metadata
- get_file_list
- read_template
- prune_file_list
- write_manifest
- get_archive_files
- setuptools.Command
- command_consumes_arguments
- distribution
- ensure_string_list
- reinitialize_command
- distutils.cmd.Command
- verbose
- force
- help
- finalized
- ensure_finalized
- dump_options
- announce
- debug_print
- ensure_string
- ensure_filename
- ensure_dirname
- get_command_name
- set_undefined_options
- get_finalized_command
- run_command
- get_sub_commands
- warn
- execute
- mkpath
- copy_file
- copy_tree
- move_file
- spawn
- make_archive
- make_file
class
build(distutils.command.build.build):
474class build(build_orig): 475 def __init__(self, dist: Distribution, build_config: BuildConfig) -> None: 476 self.build_config: BuildConfig = build_config 477 super().__init__(dist) 478 479 def finalize_options(self): 480 # print('<<< build.finalize_options is currently being run >>>') 481 try: 482 with secure_current_dir(): 483 setuptools_extensions = [ext for ext in self.distribution.ext_modules if isinstance(ext, SetuptoolsExtension)] 484 cython_extensions = [ext for ext in self.distribution.ext_modules if isinstance(ext, CythonExtension)] 485 cengal_extensions = (ext for ext in self.distribution.ext_modules if (isinstance(ext, CengalBuildExtension) and (not ext.store_as_data))) 486 cengal_result_extensions = [ext() for ext in cengal_extensions] 487 setuptools_extensions.extend([ext for ext in cengal_result_extensions if isinstance(ext, SetuptoolsExtension)]) 488 cython_extensions.extend([ext for ext in cengal_result_extensions if isinstance(ext, CythonExtension)]) 489 cengal_extensions_store_as_data = (ext for ext in self.distribution.ext_modules if (isinstance(ext, CengalBuildExtension) and ext.store_as_data)) 490 491 if self.build_config.package_build_is_in_debug_mode in environ: 492 import debugpy 493 debugpy.breakpoint() 494 495 super().finalize_options() 496 497 with secure_current_dir(): 498 package_data = self.distribution.package_data or dict() 499 500 if self.build_config.package_build_is_in_sdist_mode in environ: 501 for ext in cengal_extensions_store_as_data: 502 files = ext.sdist() 503 if files is None: 504 continue 505 506 if ext.package not in package_data: 507 package_data[ext.package] = list() 508 509 package_data[ext.package].extend(files) 510 511 result_ext_modules = list() 512 result_ext_modules.extend(cython_extensions) 513 result_ext_modules.extend(setuptools_extensions) 514 self.distribution.package_data = package_data 515 self.distribution.ext_modules = result_ext_modules 516 # from pprint import pprint 517 # print() 518 # print('<<< self.distribution.package_data: >>>') 519 # pprint(self.distribution.package_data) 520 # print() 521 # print('<<< self.distribution.ext_modules: >>>') 522 # pprint(self.distribution.ext_modules) 523 # print() 524 else: 525 for ext in cengal_extensions_store_as_data: 526 files = ext() 527 if files is None: 528 continue 529 530 if ext.package not in package_data: 531 package_data[ext.package] = list() 532 533 package_data[ext.package].extend(files) 534 535 from Cython.Build import cythonize 536 result_ext_modules = list() 537 try: 538 cwd_before_cythonize = getcwd() 539 pyx_flags_dict = prepare_pyx_flags_dict(self.build_config.additional_pyx_flags) 540 result_ext_modules.extend(remove_header_files( 541 cythonize(process_macros(cython_extensions), 542 compiler_directives={'language_level': '3'}, 543 compile_time_env = pyx_flags_dict, 544 ))) 545 except: 546 print(f'DEBUG: {cwd_before_cythonize=} | {getcwd()=}') 547 for extension in cython_extensions: 548 pprint(class_properties_values_including_overrided(extension)) 549 550 raise 551 552 result_ext_modules.extend(setuptools_extensions) 553 for ext in result_ext_modules: 554 if not hasattr(ext, 'extra_compile_args'): 555 ext.extra_compile_args = list() 556 557 if ext.extra_compile_args is None: 558 ext.extra_compile_args = list() 559 560 if 'Darwin' == OS_TYPE: 561 if cpu_info().is_x86: 562 ext.extra_compile_args += ['-arch', 'x86_64'] 563 elif cpu_info().is_arm: 564 ext.extra_compile_args += ['-arch', 'arm64'] 565 566 self.distribution.package_data = package_data 567 self.distribution.ext_modules = result_ext_modules 568 569 if self.build_config.package_build_is_in_debug_mode in environ: 570 debugpy.breakpoint() 571 print() 572 finally: 573 # print('<<< build.finalize_options is done >>>') 574 pass
Setuptools internal actions are organized using a command design pattern.
This means that each action (or group of closely related actions) executed during
the build should be implemented as a Command subclass.
These commands are abstractions and do not necessarily correspond to a command that can (or should) be executed via a terminal, in a CLI fashion (although historically they would).
When creating a new command from scratch, custom defined classes SHOULD inherit
from setuptools.Command and implement a few mandatory methods.
Between these mandatory methods, are listed:
.. method:: initialize_options(self)
Set or (reset) all options/attributes/caches used by the command
to their default values. Note that these values may be overwritten during
the build.
.. method:: finalize_options(self)
Set final values for all options/attributes used by the command.
Most of the time, each option/attribute/cache should only be set if it does not
have any value yet (e.g. ``if self.attr is None: self.attr = val``).
.. method:: run(self)
Execute the actions intended by the command.
(Side effects **SHOULD** only take place when ``run`` is executed,
for example, creating new files or writing to the terminal output).
A useful analogy for command classes is to think of them as subroutines with local
variables called "options". The options are "declared" in initialize_options()
and "defined" (given their final values, aka "finalized") in finalize_options(),
both of which must be defined by every command class. The "body" of the subroutine,
(where it does all the work) is the run() method.
Between initialize_options() and finalize_options(), setuptools may set
the values for options/attributes based on user's input (or circumstance),
which means that the implementation should be careful to not overwrite values in
finalize_options unless necessary.
Please note that other commands (or other parts of setuptools) may also overwrite
the values of the command's options/attributes multiple times during the build
process.
Therefore it is important to consistently implement initialize_options() and
finalize_options(). For example, all derived attributes (or attributes that
depend on the value of other attributes) SHOULD be recomputed in
finalize_options.
When overwriting existing commands, custom defined classes MUST abide by the same APIs implemented by the original class. They also SHOULD inherit from the original class.
build( dist: setuptools._distutils.dist.Distribution, build_config: BuildConfig)
475 def __init__(self, dist: Distribution, build_config: BuildConfig) -> None: 476 self.build_config: BuildConfig = build_config 477 super().__init__(dist)
Construct the command for dist, updating vars(self) with any keyword parameters.
build_config: BuildConfig
def
finalize_options(self):
479 def finalize_options(self): 480 # print('<<< build.finalize_options is currently being run >>>') 481 try: 482 with secure_current_dir(): 483 setuptools_extensions = [ext for ext in self.distribution.ext_modules if isinstance(ext, SetuptoolsExtension)] 484 cython_extensions = [ext for ext in self.distribution.ext_modules if isinstance(ext, CythonExtension)] 485 cengal_extensions = (ext for ext in self.distribution.ext_modules if (isinstance(ext, CengalBuildExtension) and (not ext.store_as_data))) 486 cengal_result_extensions = [ext() for ext in cengal_extensions] 487 setuptools_extensions.extend([ext for ext in cengal_result_extensions if isinstance(ext, SetuptoolsExtension)]) 488 cython_extensions.extend([ext for ext in cengal_result_extensions if isinstance(ext, CythonExtension)]) 489 cengal_extensions_store_as_data = (ext for ext in self.distribution.ext_modules if (isinstance(ext, CengalBuildExtension) and ext.store_as_data)) 490 491 if self.build_config.package_build_is_in_debug_mode in environ: 492 import debugpy 493 debugpy.breakpoint() 494 495 super().finalize_options() 496 497 with secure_current_dir(): 498 package_data = self.distribution.package_data or dict() 499 500 if self.build_config.package_build_is_in_sdist_mode in environ: 501 for ext in cengal_extensions_store_as_data: 502 files = ext.sdist() 503 if files is None: 504 continue 505 506 if ext.package not in package_data: 507 package_data[ext.package] = list() 508 509 package_data[ext.package].extend(files) 510 511 result_ext_modules = list() 512 result_ext_modules.extend(cython_extensions) 513 result_ext_modules.extend(setuptools_extensions) 514 self.distribution.package_data = package_data 515 self.distribution.ext_modules = result_ext_modules 516 # from pprint import pprint 517 # print() 518 # print('<<< self.distribution.package_data: >>>') 519 # pprint(self.distribution.package_data) 520 # print() 521 # print('<<< self.distribution.ext_modules: >>>') 522 # pprint(self.distribution.ext_modules) 523 # print() 524 else: 525 for ext in cengal_extensions_store_as_data: 526 files = ext() 527 if files is None: 528 continue 529 530 if ext.package not in package_data: 531 package_data[ext.package] = list() 532 533 package_data[ext.package].extend(files) 534 535 from Cython.Build import cythonize 536 result_ext_modules = list() 537 try: 538 cwd_before_cythonize = getcwd() 539 pyx_flags_dict = prepare_pyx_flags_dict(self.build_config.additional_pyx_flags) 540 result_ext_modules.extend(remove_header_files( 541 cythonize(process_macros(cython_extensions), 542 compiler_directives={'language_level': '3'}, 543 compile_time_env = pyx_flags_dict, 544 ))) 545 except: 546 print(f'DEBUG: {cwd_before_cythonize=} | {getcwd()=}') 547 for extension in cython_extensions: 548 pprint(class_properties_values_including_overrided(extension)) 549 550 raise 551 552 result_ext_modules.extend(setuptools_extensions) 553 for ext in result_ext_modules: 554 if not hasattr(ext, 'extra_compile_args'): 555 ext.extra_compile_args = list() 556 557 if ext.extra_compile_args is None: 558 ext.extra_compile_args = list() 559 560 if 'Darwin' == OS_TYPE: 561 if cpu_info().is_x86: 562 ext.extra_compile_args += ['-arch', 'x86_64'] 563 elif cpu_info().is_arm: 564 ext.extra_compile_args += ['-arch', 'arm64'] 565 566 self.distribution.package_data = package_data 567 self.distribution.ext_modules = result_ext_modules 568 569 if self.build_config.package_build_is_in_debug_mode in environ: 570 debugpy.breakpoint() 571 print() 572 finally: 573 # print('<<< build.finalize_options is done >>>') 574 pass
Set final values for all the options that this command supports. This is always called as late as possible, ie. after any option assignments from the command-line or from other commands have been done. Thus, this is the place to code option dependencies: if 'foo' depends on 'bar', then it is safe to set 'foo' from 'bar' as long as 'foo' still has the same value it was assigned in 'initialize_options()'.
This method must be implemented by all command classes.
Inherited Members
- distutils.command.build.build
- description
- user_options
- boolean_options
- help_options
- initialize_options
- run
- has_pure_modules
- has_c_libraries
- has_ext_modules
- has_scripts
- sub_commands
- setuptools.Command
- command_consumes_arguments
- distribution
- ensure_string_list
- reinitialize_command
- distutils.cmd.Command
- verbose
- force
- help
- finalized
- ensure_finalized
- dump_options
- announce
- debug_print
- ensure_string
- ensure_filename
- ensure_dirname
- get_command_name
- set_undefined_options
- get_finalized_command
- run_command
- get_sub_commands
- warn
- execute
- mkpath
- copy_file
- copy_tree
- move_file
- spawn
- make_archive
- make_file
class
build_ext(distutils.command.build_ext.build_ext):
577class build_ext(build_ext_orig): 578 def __init__(self, dist: Distribution, build_config: BuildConfig) -> None: 579 self.build_config: BuildConfig = build_config 580 super().__init__(dist) 581 582 def finalize_options(self): 583 with secure_current_dir(): 584 setuptools_extensions = [ext for ext in self.distribution.ext_modules if isinstance(ext, SetuptoolsExtension)] 585 cython_extensions = [ext for ext in self.distribution.ext_modules if isinstance(ext, CythonExtension)] 586 cengal_extensions = (ext for ext in self.distribution.ext_modules if (isinstance(ext, CengalBuildExtension) and (not ext.store_as_data))) 587 cengal_result_extensions = [ext() for ext in cengal_extensions] 588 setuptools_extensions.extend([ext for ext in cengal_result_extensions if isinstance(ext, SetuptoolsExtension)]) 589 cython_extensions.extend([ext for ext in cengal_result_extensions if isinstance(ext, CythonExtension)]) 590 cengal_extensions_store_as_data = (ext for ext in self.distribution.ext_modules if (isinstance(ext, CengalBuildExtension) and ext.store_as_data)) 591 592 if self.build_config.package_build_is_in_debug_mode in environ: 593 import debugpy 594 debugpy.breakpoint() 595 596 super().finalize_options() 597 598 with secure_current_dir(): 599 package_data = self.distribution.package_data or dict() 600 601 if self.build_config.package_build_is_in_sdist_mode not in environ: 602 for ext in cengal_extensions_store_as_data: 603 files = ext() 604 if files is None: 605 continue 606 607 if ext.package not in package_data: 608 package_data[ext.package] = list() 609 610 package_data[ext.package].extend(files) 611 612 from Cython.Build import cythonize 613 result_ext_modules = list() 614 try: 615 cwd_before_cythonize = getcwd() 616 pyx_flags_dict = prepare_pyx_flags_dict(self.build_config.additional_pyx_flags) 617 result_ext_modules.extend(remove_header_files( 618 cythonize(process_macros(cython_extensions), 619 compiler_directives={'language_level': '3'}, 620 compile_time_env = pyx_flags_dict, 621 ))) 622 except: 623 print(f'DEBUG: {cwd_before_cythonize=} | {getcwd()=}') 624 for extension in cython_extensions: 625 pprint(class_properties_values_including_overrided(extension)) 626 627 raise 628 629 result_ext_modules.extend(setuptools_extensions) 630 for ext in result_ext_modules: 631 if not hasattr(ext, 'extra_compile_args'): 632 ext.extra_compile_args = list() 633 634 if ext.extra_compile_args is None: 635 ext.extra_compile_args = list() 636 637 if 'Darwin' == OS_TYPE: 638 if cpu_info().is_x86: 639 ext.extra_compile_args += ['-arch', 'x86_64'] 640 elif cpu_info().is_arm: 641 ext.extra_compile_args += ['-arch', 'arm64'] 642 643 self.distribution.package_data = package_data 644 self.distribution.ext_modules = result_ext_modules 645 646 if self.build_config.package_build_is_in_debug_mode in environ: 647 debugpy.breakpoint() 648 print()
Setuptools internal actions are organized using a command design pattern.
This means that each action (or group of closely related actions) executed during
the build should be implemented as a Command subclass.
These commands are abstractions and do not necessarily correspond to a command that can (or should) be executed via a terminal, in a CLI fashion (although historically they would).
When creating a new command from scratch, custom defined classes SHOULD inherit
from setuptools.Command and implement a few mandatory methods.
Between these mandatory methods, are listed:
.. method:: initialize_options(self)
Set or (reset) all options/attributes/caches used by the command
to their default values. Note that these values may be overwritten during
the build.
.. method:: finalize_options(self)
Set final values for all options/attributes used by the command.
Most of the time, each option/attribute/cache should only be set if it does not
have any value yet (e.g. ``if self.attr is None: self.attr = val``).
.. method:: run(self)
Execute the actions intended by the command.
(Side effects **SHOULD** only take place when ``run`` is executed,
for example, creating new files or writing to the terminal output).
A useful analogy for command classes is to think of them as subroutines with local
variables called "options". The options are "declared" in initialize_options()
and "defined" (given their final values, aka "finalized") in finalize_options(),
both of which must be defined by every command class. The "body" of the subroutine,
(where it does all the work) is the run() method.
Between initialize_options() and finalize_options(), setuptools may set
the values for options/attributes based on user's input (or circumstance),
which means that the implementation should be careful to not overwrite values in
finalize_options unless necessary.
Please note that other commands (or other parts of setuptools) may also overwrite
the values of the command's options/attributes multiple times during the build
process.
Therefore it is important to consistently implement initialize_options() and
finalize_options(). For example, all derived attributes (or attributes that
depend on the value of other attributes) SHOULD be recomputed in
finalize_options.
When overwriting existing commands, custom defined classes MUST abide by the same APIs implemented by the original class. They also SHOULD inherit from the original class.
build_ext( dist: setuptools._distutils.dist.Distribution, build_config: BuildConfig)
578 def __init__(self, dist: Distribution, build_config: BuildConfig) -> None: 579 self.build_config: BuildConfig = build_config 580 super().__init__(dist)
Construct the command for dist, updating vars(self) with any keyword parameters.
build_config: BuildConfig
def
finalize_options(self):
582 def finalize_options(self): 583 with secure_current_dir(): 584 setuptools_extensions = [ext for ext in self.distribution.ext_modules if isinstance(ext, SetuptoolsExtension)] 585 cython_extensions = [ext for ext in self.distribution.ext_modules if isinstance(ext, CythonExtension)] 586 cengal_extensions = (ext for ext in self.distribution.ext_modules if (isinstance(ext, CengalBuildExtension) and (not ext.store_as_data))) 587 cengal_result_extensions = [ext() for ext in cengal_extensions] 588 setuptools_extensions.extend([ext for ext in cengal_result_extensions if isinstance(ext, SetuptoolsExtension)]) 589 cython_extensions.extend([ext for ext in cengal_result_extensions if isinstance(ext, CythonExtension)]) 590 cengal_extensions_store_as_data = (ext for ext in self.distribution.ext_modules if (isinstance(ext, CengalBuildExtension) and ext.store_as_data)) 591 592 if self.build_config.package_build_is_in_debug_mode in environ: 593 import debugpy 594 debugpy.breakpoint() 595 596 super().finalize_options() 597 598 with secure_current_dir(): 599 package_data = self.distribution.package_data or dict() 600 601 if self.build_config.package_build_is_in_sdist_mode not in environ: 602 for ext in cengal_extensions_store_as_data: 603 files = ext() 604 if files is None: 605 continue 606 607 if ext.package not in package_data: 608 package_data[ext.package] = list() 609 610 package_data[ext.package].extend(files) 611 612 from Cython.Build import cythonize 613 result_ext_modules = list() 614 try: 615 cwd_before_cythonize = getcwd() 616 pyx_flags_dict = prepare_pyx_flags_dict(self.build_config.additional_pyx_flags) 617 result_ext_modules.extend(remove_header_files( 618 cythonize(process_macros(cython_extensions), 619 compiler_directives={'language_level': '3'}, 620 compile_time_env = pyx_flags_dict, 621 ))) 622 except: 623 print(f'DEBUG: {cwd_before_cythonize=} | {getcwd()=}') 624 for extension in cython_extensions: 625 pprint(class_properties_values_including_overrided(extension)) 626 627 raise 628 629 result_ext_modules.extend(setuptools_extensions) 630 for ext in result_ext_modules: 631 if not hasattr(ext, 'extra_compile_args'): 632 ext.extra_compile_args = list() 633 634 if ext.extra_compile_args is None: 635 ext.extra_compile_args = list() 636 637 if 'Darwin' == OS_TYPE: 638 if cpu_info().is_x86: 639 ext.extra_compile_args += ['-arch', 'x86_64'] 640 elif cpu_info().is_arm: 641 ext.extra_compile_args += ['-arch', 'arm64'] 642 643 self.distribution.package_data = package_data 644 self.distribution.ext_modules = result_ext_modules 645 646 if self.build_config.package_build_is_in_debug_mode in environ: 647 debugpy.breakpoint() 648 print()
Set final values for all the options that this command supports. This is always called as late as possible, ie. after any option assignments from the command-line or from other commands have been done. Thus, this is the place to code option dependencies: if 'foo' depends on 'bar', then it is safe to set 'foo' from 'bar' as long as 'foo' still has the same value it was assigned in 'initialize_options()'.
This method must be implemented by all command classes.
Inherited Members
- distutils.command.build_ext.build_ext
- description
- sep_by
- user_options
- boolean_options
- help_options
- initialize_options
- run
- check_extensions_list
- get_source_files
- get_outputs
- build_extensions
- build_extension
- swig_sources
- find_swig
- get_ext_fullpath
- get_ext_fullname
- get_ext_filename
- get_export_symbols
- get_libraries
- setuptools.Command
- command_consumes_arguments
- distribution
- ensure_string_list
- reinitialize_command
- distutils.cmd.Command
- sub_commands
- verbose
- force
- help
- finalized
- ensure_finalized
- dump_options
- announce
- debug_print
- ensure_string
- ensure_filename
- ensure_dirname
- get_command_name
- set_undefined_options
- get_finalized_command
- run_command
- get_sub_commands
- warn
- execute
- mkpath
- copy_file
- copy_tree
- move_file
- spawn
- make_archive
- make_file