cengal.build_tools.build_extensions.versions.v_0.nim_extension
1#!/usr/bin/env python 2# coding=utf-8 3 4# Copyright © 2012-2024 ButenkoMS. All rights reserved. Contacts: <gtalk@butenkoms.space> 5# 6# Licensed under the Apache License, Version 2.0 (the "License"); 7# you may not use this file except in compliance with the License. 8# You may obtain a copy of the License at 9# 10# http://www.apache.org/licenses/LICENSE-2.0 11# 12# Unless required by applicable law or agreed to in writing, software 13# distributed under the License is distributed on an "AS IS" BASIS, 14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15# See the License for the specific language governing permissions and 16# limitations under the License. 17 18 19__all__ = [ 20 'CengalNimBuildExtension', 21] 22 23 24""" 25Module Docstring 26Docstrings: http://www.python.org/dev/peps/pep-0257/ 27""" 28 29__author__ = "ButenkoMS <gtalk@butenkoms.space>" 30__copyright__ = "Copyright © 2012-2024 ButenkoMS. All rights reserved. Contacts: <gtalk@butenkoms.space>" 31__credits__ = ["ButenkoMS <gtalk@butenkoms.space>", ] 32__license__ = "Apache License, Version 2.0" 33__version__ = "4.4.1" 34__maintainer__ = "ButenkoMS <gtalk@butenkoms.space>" 35__email__ = "gtalk@butenkoms.space" 36# __status__ = "Prototype" 37__status__ = "Development" 38# __status__ = "Production" 39 40 41# from distutils.dist import Distribution 42from os import environ 43 44from setuptools._distutils.dist import Distribution 45 46from cengal.file_system.path_manager import path_relative_to_src, RelativePath, get_relative_path_part, sep 47from cengal.file_system.directory_manager import current_src_dir, change_current_dir 48from cengal.file_system.directory_manager import filtered_file_list, FilteringType, filtered_file_list_traversal, file_list_traversal, file_list_traversal_ex, FilteringEntity 49from cengal.file_system.file_manager import current_src_file_dir, file_exists, full_ext, file_name as get_file_name, last_ext 50from cengal.build_tools.prepare_cflags import prepare_cflags, concat_cflags, prepare_compile_time_env, adjust_definition_names, \ 51 dict_of_tuples_to_dict, list_to_dict 52from cengal.introspection.inspect import get_exception, exception_to_printable_text, entity_repr_limited_try_qualname, pifrl, pdi 53from cengal.text_processing.text_processing import find_text 54from cengal.system import OS_TYPE, TEMPLATE_MODULE_NAME 55from shutil import rmtree 56from os import remove 57from os.path import splitext, normpath, join as path_join, basename, split 58from setuptools import Extension as SetuptoolsExtension 59from Cython.Distutils import Extension as CythonExtension 60from distutils.command.build import build as build_orig 61from distutils.command.build_ext import build_ext as build_ext_orig 62from setuptools.command.sdist import sdist as sdist_orig 63import json 64import importlib 65 66from os.path import isdir, exists, isfile, dirname 67 68import setuptools 69import platform 70 71from cengal.file_system.path_manager import RelativePath, get_relative_path_part 72from cengal.file_system.directory_manager import current_src_dir 73from cengal.file_system.directory_manager import file_list_traversal, FilteringEntity 74from cengal.build_tools.prepare_cflags import prepare_compile_time_flags, prepare_compile_time_env 75from cengal.os.execute import prepare_params, escape_text, escape_param, prepare_command 76from setuptools.discovery import find_package_path 77import subprocess 78from pprint import pprint 79from typing import List, Dict, Optional, Iterable, Callable, Sequence, Tuple, Union, Type, Any 80 81from .build_extensions import CengalBuildExtension 82 83 84def wrap_definition_text(text: str) -> str: 85 return text 86 87 88def wrap_definition_text_v(text: str) -> str: 89 return escape_param(text) 90 91 92def wrap_definition_text_f(text: str) -> str: 93 return f'"""{text}"""' 94 95 96def prepare_definition_value(value: Union[None, bool, int, str]) -> Union[None, str]: 97 if value is None: 98 return None 99 elif isinstance(value, bool): 100 return 'true' if value else 'false' 101 elif isinstance(value, int): 102 return str(value) 103 elif isinstance(value, str): 104 return wrap_definition_text(value) 105 else: 106 return wrap_definition_text(f'{value}') 107 108 109def prepare_definition_value_v(value: Union[None, bool, int, str]) -> Union[None, str]: 110 if value is None: 111 return None 112 elif isinstance(value, bool): 113 return 'true' if value else 'false' 114 elif isinstance(value, int): 115 return str(value) 116 elif isinstance(value, str): 117 return wrap_definition_text_v(value) 118 else: 119 return wrap_definition_text_v(f'{value}') 120 121 122def prepare_definition_value_f(value: Union[None, bool, int, str]) -> Union[None, str]: 123 if value is None: 124 return None 125 elif isinstance(value, bool): 126 return 'true' if value else 'false' 127 elif isinstance(value, (int, float)): 128 return str(value) 129 elif isinstance(value, str): 130 return wrap_definition_text_f(value) 131 else: 132 return wrap_definition_text_f(f'{value}') 133 134 135def wrap_definition_pair(name: str, value: Any) -> str: 136 return f'-d:{name}' if value is None else f'-d:{name}={value}' 137 138 139def prepare_definition(name: str, value: Optional[Union[bool, int, str]] = None) -> str: 140 return wrap_definition_pair(name, prepare_definition_value(value)) 141 142 143def prepare_definition_v(name: str, value: Optional[Union[bool, int, str]] = None) -> str: 144 return wrap_definition_pair(name, prepare_definition_value_v(value)) 145 146 147class CengalNimBuildExtension(CengalBuildExtension): 148 base_class: Optional[Type] = None 149 store_as_data: bool = True 150 151 def __init__(self, 152 module_name: str = 'main.nim', 153 flags: Optional[List[str]] = None, 154 definitions: Optional[Union[Sequence[str], Dict[str, Union[Union[None, bool, int, str], Tuple[bool, Union[None, bool, str, int]]]]]] = None, 155 additional_compilation_params: Optional[List[str]] = None, 156 definitions_module_name: str = 'compile_time_py_definitions.nim', 157 nimble_packages: Optional[List[str]] = None, 158 **kwargs) -> None: 159 self.module_name: str = module_name 160 self.flags: Optional[List[str]] = flags or list() 161 self.definitions: Optional[Union[Sequence[str], Dict[str, Union[Union[None, bool, int, str], Tuple[bool, Union[None, bool, str, int]]]]]] = definitions or dict() 162 result_flags = adjust_definition_names(list_to_dict(prepare_compile_time_flags()), 'NIMF_', 'NIMD_') # NIMF is Nim Flag; NIMD is Nim Definition 163 self.result_definitions: Optional[Dict[str, Union[None, bool, int, float, str]]] = adjust_definition_names(prepare_compile_time_env(), 'NIMF_', 'NIMD_') # NIMF is Nim Flag; NIMD is Nim Definition 164 self.result_definitions.update(result_flags) 165 self.result_definitions.update(dict_of_tuples_to_dict(list_to_dict(definitions))) 166 self.result_definitions.update(list_to_dict(flags)) 167 self.additional_compilation_params: Optional[List[str]] = additional_compilation_params 168 self.definitions_module_name: str = definitions_module_name 169 self.nimble_packages: Optional[List[str]] = nimble_packages or list() 170 super().__init__(kwargs) 171 172 def __call__(self): 173 try: 174 out_file_name: str = f'{self.module_name}.pyd' if 'Windows' == OS_TYPE else f'{self.module_name}.so' 175 print() 176 print('==================================================') 177 print(f'<<< NIMBLE PACKAGES INSTALLATION: >>>') 178 print('=======================') 179 for package_name in self.nimble_packages: 180 print(f'Installing Nimble package: {package_name}') 181 params = ['nimble', 'install', package_name, ' --accept'] 182 result = subprocess.run(params, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) 183 print(result.stdout) 184 185 print('==================================================') 186 187 print() 188 print('==================================================') 189 print(f'<<< NIM COMPILATION: {self.module_name} -> {out_file_name} >>>') 190 print('=======================') 191 params = ['nim', 'c', '--forceBuild:on', '--app:lib', f'--out:{out_file_name}', '--threads:on'] 192 params_v = ['nim', 'c', '--forceBuild:on', '--app:lib', f'--out:{out_file_name}', '--threads:on'] 193 for name, value in self.result_definitions.items(): 194 params.append(prepare_definition(name, value)) 195 params_v.append(prepare_definition_v(name, value)) 196 197 if self.additional_compilation_params: 198 params.extend(self.additional_compilation_params) 199 params_v.extend(self.additional_compilation_params) 200 201 if 'Windows' == OS_TYPE: 202 params.extend(['--tlsEmulation:off', '--passL:-static', self.module_name]) 203 params_v.extend(['--tlsEmulation:off', '--passL:-static', self.module_name]) 204 else: 205 params.extend([self.module_name,]) 206 params_v.extend([self.module_name,]) 207 208 with change_current_dir(self.dir_path): 209 self._ensure_gitignore() 210 self._generate_definitions_module() 211 print('> NIM compiler command line:') 212 print(prepare_command(params_v[0], params_v[1:])) 213 print('> NIM compiler params:') 214 pprint(params) 215 result = subprocess.run(params, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) 216 217 successed: bool = find_text(result.stdout, '[SuccessX]') is not None 218 print(f'{successed=}') 219 result_str = result.stdout 220 print('> NIM compiler output:') 221 print(result_str) 222 223 exported_files_list = [out_file_name,] 224 if successed: 225 print('> Exported files:') 226 print(exported_files_list) 227 228 print('==================================================') 229 return exported_files_list if successed else None 230 except: 231 print('==================================================') 232 print('!!! NIM COMPILATION EXCEPTION !!!') 233 print('==================================================') 234 print(exception_to_printable_text(get_exception())) 235 print('==================================================') 236 return None 237 238 def sdist(self) -> Optional[List[str]]: 239 def filter(entity: FilteringEntity, data: Any): 240 if FilteringEntity.filename == entity: 241 dirpath, filename = data 242 if filename in { 243 '__init__.py', 244 '__x__build_config.py', 245 '__build_config.py', 246 }: 247 return False 248 249 if last_ext(filename) in {'so', 'dylib', 'pyd', 'dll', 'py', 'pyw', 'gitignore', 'pyc'}: 250 return False 251 252 return True 253 elif FilteringEntity.dirname == entity: 254 dirpath, dirname = data 255 if dirname in { 256 '__pycache__', 257 }: 258 return False 259 260 return True 261 elif FilteringEntity.dirpath == entity: 262 dirpath, dirnames, filenames = data 263 dirpath_basename = basename(dirpath) 264 if dirpath_basename in { 265 '__pycache__', 266 }: 267 return False 268 269 return True 270 elif FilteringEntity.aggregated == entity: 271 result_full_file_names: List[str] = list() 272 for dirpath, new_dirnames, new_filenames in data: 273 for file_name in new_filenames: 274 result_full_file_names.append(path_join(dirpath, file_name)) 275 276 return result_full_file_names 277 else: 278 raise NotImplementedError 279 280 result_full_file_names: List[str] = file_list_traversal_ex(self.dir_path, filter, True) 281 adjusted_exported_files_list = list() 282 for file_path in result_full_file_names: 283 adjusted_exported_files_list.append(get_relative_path_part(file_path, self.dir_path)) 284 285 return adjusted_exported_files_list 286 287 def _ensure_gitignore(self): 288 gitignore_path = self.dir_path_rel('.gitignore') 289 if file_exists(gitignore_path): 290 with open(gitignore_path, 'r+t') as f: 291 content = f.read() 292 if f'{self.module_name}.pyd' not in content: 293 f.write(f'{self.module_name}.pyd\n') 294 295 if f'{self.module_name}.so' not in content: 296 f.write(f'{self.module_name}.so\n') 297 298 if f'{self.module_name}.dll' not in content: 299 f.write(f'{self.module_name}.dll\n') 300 301 if f'{self.module_name}.dylib' not in content: 302 f.write(f'{self.module_name}.dylib\n') 303 304 if self.definitions_module_name not in content: 305 f.write(f'{self.definitions_module_name}\n') 306 else: 307 with open(gitignore_path, 'xt') as f: 308 f.write(f'{self.module_name}.pyd\n') 309 f.write(f'{self.module_name}.so\n') 310 f.write(f'{self.module_name}.dll\n') 311 f.write(f'{self.module_name}.dylib\n') 312 f.write(f'{self.definitions_module_name}\n') 313 314 def _generate_definitions_module(self): 315 with open(self.dir_path_rel(self.definitions_module_name), 'wt') as f: 316 f.write(f'# {self.definitions_module_name}\n\n') 317 for name, value in self.result_definitions.items(): 318 prepared_value = prepare_definition_value_f(value) 319 if prepared_value is None: 320 continue 321 322 f.write(f'const {name}* = {prepared_value}\n')
class
CengalNimBuildExtension(cengal.build_tools.build_extensions.versions.v_0.build_extensions.CengalBuildExtension):
148class CengalNimBuildExtension(CengalBuildExtension): 149 base_class: Optional[Type] = None 150 store_as_data: bool = True 151 152 def __init__(self, 153 module_name: str = 'main.nim', 154 flags: Optional[List[str]] = None, 155 definitions: Optional[Union[Sequence[str], Dict[str, Union[Union[None, bool, int, str], Tuple[bool, Union[None, bool, str, int]]]]]] = None, 156 additional_compilation_params: Optional[List[str]] = None, 157 definitions_module_name: str = 'compile_time_py_definitions.nim', 158 nimble_packages: Optional[List[str]] = None, 159 **kwargs) -> None: 160 self.module_name: str = module_name 161 self.flags: Optional[List[str]] = flags or list() 162 self.definitions: Optional[Union[Sequence[str], Dict[str, Union[Union[None, bool, int, str], Tuple[bool, Union[None, bool, str, int]]]]]] = definitions or dict() 163 result_flags = adjust_definition_names(list_to_dict(prepare_compile_time_flags()), 'NIMF_', 'NIMD_') # NIMF is Nim Flag; NIMD is Nim Definition 164 self.result_definitions: Optional[Dict[str, Union[None, bool, int, float, str]]] = adjust_definition_names(prepare_compile_time_env(), 'NIMF_', 'NIMD_') # NIMF is Nim Flag; NIMD is Nim Definition 165 self.result_definitions.update(result_flags) 166 self.result_definitions.update(dict_of_tuples_to_dict(list_to_dict(definitions))) 167 self.result_definitions.update(list_to_dict(flags)) 168 self.additional_compilation_params: Optional[List[str]] = additional_compilation_params 169 self.definitions_module_name: str = definitions_module_name 170 self.nimble_packages: Optional[List[str]] = nimble_packages or list() 171 super().__init__(kwargs) 172 173 def __call__(self): 174 try: 175 out_file_name: str = f'{self.module_name}.pyd' if 'Windows' == OS_TYPE else f'{self.module_name}.so' 176 print() 177 print('==================================================') 178 print(f'<<< NIMBLE PACKAGES INSTALLATION: >>>') 179 print('=======================') 180 for package_name in self.nimble_packages: 181 print(f'Installing Nimble package: {package_name}') 182 params = ['nimble', 'install', package_name, ' --accept'] 183 result = subprocess.run(params, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) 184 print(result.stdout) 185 186 print('==================================================') 187 188 print() 189 print('==================================================') 190 print(f'<<< NIM COMPILATION: {self.module_name} -> {out_file_name} >>>') 191 print('=======================') 192 params = ['nim', 'c', '--forceBuild:on', '--app:lib', f'--out:{out_file_name}', '--threads:on'] 193 params_v = ['nim', 'c', '--forceBuild:on', '--app:lib', f'--out:{out_file_name}', '--threads:on'] 194 for name, value in self.result_definitions.items(): 195 params.append(prepare_definition(name, value)) 196 params_v.append(prepare_definition_v(name, value)) 197 198 if self.additional_compilation_params: 199 params.extend(self.additional_compilation_params) 200 params_v.extend(self.additional_compilation_params) 201 202 if 'Windows' == OS_TYPE: 203 params.extend(['--tlsEmulation:off', '--passL:-static', self.module_name]) 204 params_v.extend(['--tlsEmulation:off', '--passL:-static', self.module_name]) 205 else: 206 params.extend([self.module_name,]) 207 params_v.extend([self.module_name,]) 208 209 with change_current_dir(self.dir_path): 210 self._ensure_gitignore() 211 self._generate_definitions_module() 212 print('> NIM compiler command line:') 213 print(prepare_command(params_v[0], params_v[1:])) 214 print('> NIM compiler params:') 215 pprint(params) 216 result = subprocess.run(params, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) 217 218 successed: bool = find_text(result.stdout, '[SuccessX]') is not None 219 print(f'{successed=}') 220 result_str = result.stdout 221 print('> NIM compiler output:') 222 print(result_str) 223 224 exported_files_list = [out_file_name,] 225 if successed: 226 print('> Exported files:') 227 print(exported_files_list) 228 229 print('==================================================') 230 return exported_files_list if successed else None 231 except: 232 print('==================================================') 233 print('!!! NIM COMPILATION EXCEPTION !!!') 234 print('==================================================') 235 print(exception_to_printable_text(get_exception())) 236 print('==================================================') 237 return None 238 239 def sdist(self) -> Optional[List[str]]: 240 def filter(entity: FilteringEntity, data: Any): 241 if FilteringEntity.filename == entity: 242 dirpath, filename = data 243 if filename in { 244 '__init__.py', 245 '__x__build_config.py', 246 '__build_config.py', 247 }: 248 return False 249 250 if last_ext(filename) in {'so', 'dylib', 'pyd', 'dll', 'py', 'pyw', 'gitignore', 'pyc'}: 251 return False 252 253 return True 254 elif FilteringEntity.dirname == entity: 255 dirpath, dirname = data 256 if dirname in { 257 '__pycache__', 258 }: 259 return False 260 261 return True 262 elif FilteringEntity.dirpath == entity: 263 dirpath, dirnames, filenames = data 264 dirpath_basename = basename(dirpath) 265 if dirpath_basename in { 266 '__pycache__', 267 }: 268 return False 269 270 return True 271 elif FilteringEntity.aggregated == entity: 272 result_full_file_names: List[str] = list() 273 for dirpath, new_dirnames, new_filenames in data: 274 for file_name in new_filenames: 275 result_full_file_names.append(path_join(dirpath, file_name)) 276 277 return result_full_file_names 278 else: 279 raise NotImplementedError 280 281 result_full_file_names: List[str] = file_list_traversal_ex(self.dir_path, filter, True) 282 adjusted_exported_files_list = list() 283 for file_path in result_full_file_names: 284 adjusted_exported_files_list.append(get_relative_path_part(file_path, self.dir_path)) 285 286 return adjusted_exported_files_list 287 288 def _ensure_gitignore(self): 289 gitignore_path = self.dir_path_rel('.gitignore') 290 if file_exists(gitignore_path): 291 with open(gitignore_path, 'r+t') as f: 292 content = f.read() 293 if f'{self.module_name}.pyd' not in content: 294 f.write(f'{self.module_name}.pyd\n') 295 296 if f'{self.module_name}.so' not in content: 297 f.write(f'{self.module_name}.so\n') 298 299 if f'{self.module_name}.dll' not in content: 300 f.write(f'{self.module_name}.dll\n') 301 302 if f'{self.module_name}.dylib' not in content: 303 f.write(f'{self.module_name}.dylib\n') 304 305 if self.definitions_module_name not in content: 306 f.write(f'{self.definitions_module_name}\n') 307 else: 308 with open(gitignore_path, 'xt') as f: 309 f.write(f'{self.module_name}.pyd\n') 310 f.write(f'{self.module_name}.so\n') 311 f.write(f'{self.module_name}.dll\n') 312 f.write(f'{self.module_name}.dylib\n') 313 f.write(f'{self.definitions_module_name}\n') 314 315 def _generate_definitions_module(self): 316 with open(self.dir_path_rel(self.definitions_module_name), 'wt') as f: 317 f.write(f'# {self.definitions_module_name}\n\n') 318 for name, value in self.result_definitions.items(): 319 prepared_value = prepare_definition_value_f(value) 320 if prepared_value is None: 321 continue 322 323 f.write(f'const {name}* = {prepared_value}\n')
CengalNimBuildExtension( module_name: str = 'main.nim', flags: Union[List[str], NoneType] = None, definitions: Union[Sequence[str], Dict[str, Union[NoneType, bool, int, str, Tuple[bool, Union[NoneType, bool, str, int]]]], NoneType] = None, additional_compilation_params: Union[List[str], NoneType] = None, definitions_module_name: str = 'compile_time_py_definitions.nim', nimble_packages: Union[List[str], NoneType] = None, **kwargs)
152 def __init__(self, 153 module_name: str = 'main.nim', 154 flags: Optional[List[str]] = None, 155 definitions: Optional[Union[Sequence[str], Dict[str, Union[Union[None, bool, int, str], Tuple[bool, Union[None, bool, str, int]]]]]] = None, 156 additional_compilation_params: Optional[List[str]] = None, 157 definitions_module_name: str = 'compile_time_py_definitions.nim', 158 nimble_packages: Optional[List[str]] = None, 159 **kwargs) -> None: 160 self.module_name: str = module_name 161 self.flags: Optional[List[str]] = flags or list() 162 self.definitions: Optional[Union[Sequence[str], Dict[str, Union[Union[None, bool, int, str], Tuple[bool, Union[None, bool, str, int]]]]]] = definitions or dict() 163 result_flags = adjust_definition_names(list_to_dict(prepare_compile_time_flags()), 'NIMF_', 'NIMD_') # NIMF is Nim Flag; NIMD is Nim Definition 164 self.result_definitions: Optional[Dict[str, Union[None, bool, int, float, str]]] = adjust_definition_names(prepare_compile_time_env(), 'NIMF_', 'NIMD_') # NIMF is Nim Flag; NIMD is Nim Definition 165 self.result_definitions.update(result_flags) 166 self.result_definitions.update(dict_of_tuples_to_dict(list_to_dict(definitions))) 167 self.result_definitions.update(list_to_dict(flags)) 168 self.additional_compilation_params: Optional[List[str]] = additional_compilation_params 169 self.definitions_module_name: str = definitions_module_name 170 self.nimble_packages: Optional[List[str]] = nimble_packages or list() 171 super().__init__(kwargs)
definitions: Union[Sequence[str], Dict[str, Union[NoneType, bool, int, str, Tuple[bool, Union[NoneType, bool, str, int]]]], NoneType]
def
sdist(self) -> Union[List[str], NoneType]:
239 def sdist(self) -> Optional[List[str]]: 240 def filter(entity: FilteringEntity, data: Any): 241 if FilteringEntity.filename == entity: 242 dirpath, filename = data 243 if filename in { 244 '__init__.py', 245 '__x__build_config.py', 246 '__build_config.py', 247 }: 248 return False 249 250 if last_ext(filename) in {'so', 'dylib', 'pyd', 'dll', 'py', 'pyw', 'gitignore', 'pyc'}: 251 return False 252 253 return True 254 elif FilteringEntity.dirname == entity: 255 dirpath, dirname = data 256 if dirname in { 257 '__pycache__', 258 }: 259 return False 260 261 return True 262 elif FilteringEntity.dirpath == entity: 263 dirpath, dirnames, filenames = data 264 dirpath_basename = basename(dirpath) 265 if dirpath_basename in { 266 '__pycache__', 267 }: 268 return False 269 270 return True 271 elif FilteringEntity.aggregated == entity: 272 result_full_file_names: List[str] = list() 273 for dirpath, new_dirnames, new_filenames in data: 274 for file_name in new_filenames: 275 result_full_file_names.append(path_join(dirpath, file_name)) 276 277 return result_full_file_names 278 else: 279 raise NotImplementedError 280 281 result_full_file_names: List[str] = file_list_traversal_ex(self.dir_path, filter, True) 282 adjusted_exported_files_list = list() 283 for file_path in result_full_file_names: 284 adjusted_exported_files_list.append(get_relative_path_part(file_path, self.dir_path)) 285 286 return adjusted_exported_files_list
Inherited Members
- cengal.build_tools.build_extensions.versions.v_0.build_extensions.CengalBuildExtension
- kwargs
- path
- dir_path
- dir_path_rel
- module_import_str
- name
- package
- files