cengal.build_tools.gather_docs.versions.v_0.gather_docs

View Source

  1#!/usr/bin/env python
  2# coding=utf-8
  3
  4# Copyright © 2012-2024 ButenkoMS. All rights reserved. Contacts: <gtalk@butenkoms.space>
  5# 
  6# Licensed under the Apache License, Version 2.0 (the "License");
  7# you may not use this file except in compliance with the License.
  8# You may obtain a copy of the License at
  9# 
 10#     http://www.apache.org/licenses/LICENSE-2.0
 11# 
 12# Unless required by applicable law or agreed to in writing, software
 13# distributed under the License is distributed on an "AS IS" BASIS,
 14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 15# See the License for the specific language governing permissions and
 16# limitations under the License.
 17
 18
 19__all__ = [
 20    'auto_gen', 
 21    'gather', 
 22    'merge_docs', 
 23    'gather_docs', 
 24]
 25
 26
 27"""
 28Module Docstring
 29Docstrings: http://www.python.org/dev/peps/pep-0257/
 30"""
 31
 32__author__ = "ButenkoMS <gtalk@butenkoms.space>"
 33__copyright__ = "Copyright © 2012-2024 ButenkoMS. All rights reserved. Contacts: <gtalk@butenkoms.space>"
 34__credits__ = ["ButenkoMS <gtalk@butenkoms.space>", ]
 35__license__ = "Apache License, Version 2.0"
 36__version__ = "4.4.1"
 37__maintainer__ = "ButenkoMS <gtalk@butenkoms.space>"
 38__email__ = "gtalk@butenkoms.space"
 39# __status__ = "Prototype"
 40__status__ = "Development"
 41# __status__ = "Production"
 42
 43
 44from cengal.file_system.directory_manager import FilteringEntity, file_list_traversal, file_list_traversal_ex, ensure_empty_dir, ensure_dir
 45from cengal.file_system.file_manager import last_ext
 46from cengal.file_system.path_manager import relative_to_src, get_relative_path_part, RelativePath
 47from shutil import copyfile
 48from os.path import join as path_join, dirname, basename
 49from pprint import pprint
 50from typing import Any, List, Set, Optional
 51from pathlib import Path
 52import shutil
 53
 54from pdoc import pdoc
 55from pdoc import render
 56
 57
 58def auto_gen(repo_root_dir, root_dir, docs_dir, subignore_rel_path: Optional[List[str]] = None):
 59    subignore_rel_path = subignore_rel_path or list()
 60    repo_root_dir_rel = RelativePath(repo_root_dir)
 61    docs_dir_rel = RelativePath(docs_dir)
 62    src_dir_name = basename(root_dir)
 63    def py_filter(entity: FilteringEntity, data: Any):
 64        if FilteringEntity.filename == entity:
 65            dirpath, filename = data
 66            if filename in {
 67                '__init__.py', 
 68                '_template_submodule.py', 
 69                '__requirements__.py', 
 70                '_test__template_submodule.py', 
 71                '__x__build_config.py', 
 72                '__build_config.py', 
 73            }:
 74                return False
 75            
 76            if filename.startswith('test_'):
 77                return False
 78
 79            return last_ext(filename) in {'py', 'pyw'}
 80        elif FilteringEntity.dirname == entity:
 81            dirpath, dirname = data
 82            if dirname in {
 83                '_template_module', 
 84                '_template_submodule', 
 85                'compilable', 
 86                'data', 
 87                'development', 
 88                'data', 
 89                'docs', 
 90                'tests', 
 91                'package', 
 92                'benchmarks', 
 93                '__pycache__', 
 94                }:
 95                return False
 96
 97            return True
 98        elif FilteringEntity.dirpath == entity:
 99            dirpath, dirnames, filenames = data
100            dirpath_basename = basename(dirpath)
101            if dirpath_basename in {
102                '_template_module', 
103                '_template_submodule', 
104                'compilable', 
105                'data', 
106                'development', 
107                'data', 
108                'docs', 
109                'tests', 
110                'package', 
111                'benchmarks', 
112                '__pycache__', 
113                }:
114                return False
115            
116            return True
117        elif FilteringEntity.aggregated == entity:
118            return data
119        else:
120            raise NotImplementedError
121    
122    subignore = set([repo_root_dir_rel(rel_path) for rel_path in subignore_rel_path])
123    result = file_list_traversal_ex(root_dir, py_filter, True, subignore=subignore)
124    gen_from_to = dict()
125    gen_from = set()
126    for dirpath, dirnames, filenames in result:
127        for filename in filenames:
128            src_path = path_join(dirpath, filename)
129            gen_from.add(src_path)
130    
131    pprint(gen_from)
132    dest_root_dir = docs_dir_rel(f'Auto-generated')
133    print(dest_root_dir)
134    ensure_empty_dir(dest_root_dir)
135    out = Path(dest_root_dir)
136
137    template_directory = relative_to_src()('data/pdoc-template')
138    # render.configure(template_directory=template_directory)
139    render.configure(docformat='markdown', search=False, template_directory=template_directory)
140    for src_path in gen_from:
141        try:
142            print(f'Gen for <{src_path}> into {dest_root_dir}:')
143            pdoc(src_path, output_directory=out)
144        except:
145            pass
146    
147    print(f'Removing unnecessary files')
148    unnecessary_file_names: Set[str] = {
149        'index.html',
150        'search.js',
151    }
152    for file_name in unnecessary_file_names:
153        full_file_name = out / file_name
154        if full_file_name.exists():
155            full_file_name.unlink()
156
157    print(f'Renameing from "*.html" to "*.md"')
158    for f in out.glob("**/*.html"):
159        print(f'\tRenaming <{f}> -> "*.md"')
160        f.rename(f.with_suffix(".md"))
161    
162    print('Done.')
163
164
165def gather(repo_root_dir, root_dir, docs_dir, subignore_rel_path: Optional[List[str]] = None):
166    subignore_rel_path = subignore_rel_path or list()
167    repo_root_dir_rel = RelativePath(repo_root_dir)
168    docs_dir_rel = RelativePath(docs_dir)
169    src_dir_name = basename(root_dir)
170    def md_filter(entity: FilteringEntity, data: Any):
171        if FilteringEntity.filename == entity:
172            dirpath, filename = data
173            return 'md' == last_ext(filename)
174        elif FilteringEntity.dirname == entity:
175            dirpath, dirname = data
176            if dirname in {
177                '_template_module', 
178                '_template_submodule', 
179                'tests', 
180                '__pycache__', 
181                }:
182                return False
183
184            return True
185        elif FilteringEntity.dirpath == entity:
186            dirpath, dirnames, filenames = data
187            dirpath_basename = basename(dirpath)
188            if dirpath_basename in {
189                '_template_module', 
190                '_template_submodule', 
191                'tests', 
192                '__pycache__', 
193                }:
194                return False
195            
196            return True
197        elif FilteringEntity.aggregated == entity:
198            return data
199        else:
200            raise NotImplementedError
201    
202    subignore = set([repo_root_dir_rel(rel_path) for rel_path in subignore_rel_path])
203    result = file_list_traversal_ex(root_dir, md_filter, True, subignore=subignore)
204    copy_from_to = dict()
205    for dirpath, dirnames, filenames in result:
206        for filename in filenames:
207            src_path = path_join(dirpath, filename)
208            rel_path_part = get_relative_path_part(dirpath, root_dir)
209            dest_path = path_join(src_dir_name, rel_path_part, filename)
210            copy_from_to[src_path] = docs_dir_rel(dest_path)
211    
212    pprint(copy_from_to)
213    dest_root_dir = docs_dir_rel(src_dir_name)
214    print(dest_root_dir)
215    ensure_empty_dir(dest_root_dir)
216    for src_path, dest_path in copy_from_to.items():
217        print(f'{src_path} -> {dest_path}')
218        dest_file_dir = dirname(dest_path)
219        print(f'{dest_file_dir}')
220        ensure_dir(dest_file_dir)
221        copyfile(src_path, dest_path)
222
223
224def merge_docs(repo_root_dir, root_dir, docs_dir):
225    docs_dir_rel = RelativePath(docs_dir)
226    src_dir_name = basename(root_dir)
227    docs_lib_dir = docs_dir_rel(src_dir_name)
228    docs_gen_dir = docs_dir_rel('Auto-generated')
229    docs_gen_dir_rel = RelativePath(docs_gen_dir)
230    docs_gen_lib_dir = docs_gen_dir_rel(src_dir_name)
231    def md_filter(entity: FilteringEntity, data: Any):
232        if FilteringEntity.filename == entity:
233            dirpath, filename = data
234            return 'md' == last_ext(filename)
235        elif FilteringEntity.dirname == entity:
236            dirpath, dirname = data
237            return True
238        elif FilteringEntity.dirpath == entity:
239            return True
240        elif FilteringEntity.aggregated == entity:
241            return data
242        else:
243            raise NotImplementedError
244    
245    result = file_list_traversal(docs_gen_lib_dir, md_filter, True)
246    copy_from_to = dict()
247    for dirpath, dirnames, filenames in result:
248        for filename in filenames:
249            src_path = path_join(dirpath, filename)
250            rel_path_part = get_relative_path_part(dirpath, docs_gen_lib_dir)
251            dest_path = path_join(src_dir_name, rel_path_part, filename)
252            copy_from_to[src_path] = docs_dir_rel(dest_path)
253    
254    pprint(copy_from_to)
255    dest_root_dir = docs_dir_rel(src_dir_name)
256    print(dest_root_dir)
257    ensure_dir(dest_root_dir)
258    for src_path, dest_path in copy_from_to.items():
259        print(f'{src_path} -> {dest_path}')
260        dest_file_dir = dirname(dest_path)
261        print(f'{dest_file_dir}')
262        ensure_dir(dest_file_dir)
263        copyfile(src_path, dest_path)
264    
265    print(f'Removing unnecessary generated docs from "{docs_gen_dir}"')
266    shutil.rmtree(docs_gen_dir)
267    print('Done.')
268
269
270def gather_docs(repo_root_dir, root_dir, docs_dir, modules_to_ignore: Optional[List[str]] = None):
271    auto_gen(repo_root_dir, root_dir, docs_dir, modules_to_ignore)
272    gather(repo_root_dir, root_dir, docs_dir, modules_to_ignore)
273    merge_docs(repo_root_dir, root_dir, docs_dir)
274
275
276if '__main__' == __name__:
277    cengal_repo_root_dir = relative_to_src()('../../../../..')
278    cengal_docs_dir = RelativePath(cengal_repo_root_dir)('docs')
279    cengal_src_root_dir = relative_to_src()('../../../..')
280    subignore_rel_path = {
281        'cengal/_examples', 
282        'cengal/_template_module', 
283        'cengal/code_flow_control/none_or', 
284        'cengal/cross_version/console_print', 
285        'cengal/help_tools', 
286        'cengal/os/help_tools', 
287        'cengal/parallel_execution/coroutines/coro_standard_services/cpu_tick_count_per_second', 
288        'cengal/testing_lib', 
289        'cengal/text_processing/docten', 
290        'cengal/time_management/server_clock', 
291        'cengal/universal_parser', 
292        'cengal/upk_helping_tools', 
293        'cengal/user_interface/console/chooser', 
294    }
295    auto_gen(cengal_repo_root_dir, cengal_src_root_dir, cengal_docs_dir, subignore_rel_path)
296    gather(cengal_repo_root_dir, cengal_src_root_dir, cengal_docs_dir, subignore_rel_path)
297    merge_docs(cengal_repo_root_dir, cengal_src_root_dir, cengal_docs_dir)

def auto_gen( repo_root_dir, root_dir, docs_dir, subignore_rel_path: typing.Union[typing.List[str], NoneType] = None): View Source

 59def auto_gen(repo_root_dir, root_dir, docs_dir, subignore_rel_path: Optional[List[str]] = None):
 60    subignore_rel_path = subignore_rel_path or list()
 61    repo_root_dir_rel = RelativePath(repo_root_dir)
 62    docs_dir_rel = RelativePath(docs_dir)
 63    src_dir_name = basename(root_dir)
 64    def py_filter(entity: FilteringEntity, data: Any):
 65        if FilteringEntity.filename == entity:
 66            dirpath, filename = data
 67            if filename in {
 68                '__init__.py', 
 69                '_template_submodule.py', 
 70                '__requirements__.py', 
 71                '_test__template_submodule.py', 
 72                '__x__build_config.py', 
 73                '__build_config.py', 
 74            }:
 75                return False
 76            
 77            if filename.startswith('test_'):
 78                return False
 79
 80            return last_ext(filename) in {'py', 'pyw'}
 81        elif FilteringEntity.dirname == entity:
 82            dirpath, dirname = data
 83            if dirname in {
 84                '_template_module', 
 85                '_template_submodule', 
 86                'compilable', 
 87                'data', 
 88                'development', 
 89                'data', 
 90                'docs', 
 91                'tests', 
 92                'package', 
 93                'benchmarks', 
 94                '__pycache__', 
 95                }:
 96                return False
 97
 98            return True
 99        elif FilteringEntity.dirpath == entity:
100            dirpath, dirnames, filenames = data
101            dirpath_basename = basename(dirpath)
102            if dirpath_basename in {
103                '_template_module', 
104                '_template_submodule', 
105                'compilable', 
106                'data', 
107                'development', 
108                'data', 
109                'docs', 
110                'tests', 
111                'package', 
112                'benchmarks', 
113                '__pycache__', 
114                }:
115                return False
116            
117            return True
118        elif FilteringEntity.aggregated == entity:
119            return data
120        else:
121            raise NotImplementedError
122    
123    subignore = set([repo_root_dir_rel(rel_path) for rel_path in subignore_rel_path])
124    result = file_list_traversal_ex(root_dir, py_filter, True, subignore=subignore)
125    gen_from_to = dict()
126    gen_from = set()
127    for dirpath, dirnames, filenames in result:
128        for filename in filenames:
129            src_path = path_join(dirpath, filename)
130            gen_from.add(src_path)
131    
132    pprint(gen_from)
133    dest_root_dir = docs_dir_rel(f'Auto-generated')
134    print(dest_root_dir)
135    ensure_empty_dir(dest_root_dir)
136    out = Path(dest_root_dir)
137
138    template_directory = relative_to_src()('data/pdoc-template')
139    # render.configure(template_directory=template_directory)
140    render.configure(docformat='markdown', search=False, template_directory=template_directory)
141    for src_path in gen_from:
142        try:
143            print(f'Gen for <{src_path}> into {dest_root_dir}:')
144            pdoc(src_path, output_directory=out)
145        except:
146            pass
147    
148    print(f'Removing unnecessary files')
149    unnecessary_file_names: Set[str] = {
150        'index.html',
151        'search.js',
152    }
153    for file_name in unnecessary_file_names:
154        full_file_name = out / file_name
155        if full_file_name.exists():
156            full_file_name.unlink()
157
158    print(f'Renameing from "*.html" to "*.md"')
159    for f in out.glob("**/*.html"):
160        print(f'\tRenaming <{f}> -> "*.md"')
161        f.rename(f.with_suffix(".md"))
162    
163    print('Done.')

def gather( repo_root_dir, root_dir, docs_dir, subignore_rel_path: typing.Union[typing.List[str], NoneType] = None): View Source

166def gather(repo_root_dir, root_dir, docs_dir, subignore_rel_path: Optional[List[str]] = None):
167    subignore_rel_path = subignore_rel_path or list()
168    repo_root_dir_rel = RelativePath(repo_root_dir)
169    docs_dir_rel = RelativePath(docs_dir)
170    src_dir_name = basename(root_dir)
171    def md_filter(entity: FilteringEntity, data: Any):
172        if FilteringEntity.filename == entity:
173            dirpath, filename = data
174            return 'md' == last_ext(filename)
175        elif FilteringEntity.dirname == entity:
176            dirpath, dirname = data
177            if dirname in {
178                '_template_module', 
179                '_template_submodule', 
180                'tests', 
181                '__pycache__', 
182                }:
183                return False
184
185            return True
186        elif FilteringEntity.dirpath == entity:
187            dirpath, dirnames, filenames = data
188            dirpath_basename = basename(dirpath)
189            if dirpath_basename in {
190                '_template_module', 
191                '_template_submodule', 
192                'tests', 
193                '__pycache__', 
194                }:
195                return False
196            
197            return True
198        elif FilteringEntity.aggregated == entity:
199            return data
200        else:
201            raise NotImplementedError
202    
203    subignore = set([repo_root_dir_rel(rel_path) for rel_path in subignore_rel_path])
204    result = file_list_traversal_ex(root_dir, md_filter, True, subignore=subignore)
205    copy_from_to = dict()
206    for dirpath, dirnames, filenames in result:
207        for filename in filenames:
208            src_path = path_join(dirpath, filename)
209            rel_path_part = get_relative_path_part(dirpath, root_dir)
210            dest_path = path_join(src_dir_name, rel_path_part, filename)
211            copy_from_to[src_path] = docs_dir_rel(dest_path)
212    
213    pprint(copy_from_to)
214    dest_root_dir = docs_dir_rel(src_dir_name)
215    print(dest_root_dir)
216    ensure_empty_dir(dest_root_dir)
217    for src_path, dest_path in copy_from_to.items():
218        print(f'{src_path} -> {dest_path}')
219        dest_file_dir = dirname(dest_path)
220        print(f'{dest_file_dir}')
221        ensure_dir(dest_file_dir)
222        copyfile(src_path, dest_path)

def merge_docs(repo_root_dir, root_dir, docs_dir): View Source

225def merge_docs(repo_root_dir, root_dir, docs_dir):
226    docs_dir_rel = RelativePath(docs_dir)
227    src_dir_name = basename(root_dir)
228    docs_lib_dir = docs_dir_rel(src_dir_name)
229    docs_gen_dir = docs_dir_rel('Auto-generated')
230    docs_gen_dir_rel = RelativePath(docs_gen_dir)
231    docs_gen_lib_dir = docs_gen_dir_rel(src_dir_name)
232    def md_filter(entity: FilteringEntity, data: Any):
233        if FilteringEntity.filename == entity:
234            dirpath, filename = data
235            return 'md' == last_ext(filename)
236        elif FilteringEntity.dirname == entity:
237            dirpath, dirname = data
238            return True
239        elif FilteringEntity.dirpath == entity:
240            return True
241        elif FilteringEntity.aggregated == entity:
242            return data
243        else:
244            raise NotImplementedError
245    
246    result = file_list_traversal(docs_gen_lib_dir, md_filter, True)
247    copy_from_to = dict()
248    for dirpath, dirnames, filenames in result:
249        for filename in filenames:
250            src_path = path_join(dirpath, filename)
251            rel_path_part = get_relative_path_part(dirpath, docs_gen_lib_dir)
252            dest_path = path_join(src_dir_name, rel_path_part, filename)
253            copy_from_to[src_path] = docs_dir_rel(dest_path)
254    
255    pprint(copy_from_to)
256    dest_root_dir = docs_dir_rel(src_dir_name)
257    print(dest_root_dir)
258    ensure_dir(dest_root_dir)
259    for src_path, dest_path in copy_from_to.items():
260        print(f'{src_path} -> {dest_path}')
261        dest_file_dir = dirname(dest_path)
262        print(f'{dest_file_dir}')
263        ensure_dir(dest_file_dir)
264        copyfile(src_path, dest_path)
265    
266    print(f'Removing unnecessary generated docs from "{docs_gen_dir}"')
267    shutil.rmtree(docs_gen_dir)
268    print('Done.')

def gather_docs( repo_root_dir, root_dir, docs_dir, modules_to_ignore: typing.Union[typing.List[str], NoneType] = None): View Source

271def gather_docs(repo_root_dir, root_dir, docs_dir, modules_to_ignore: Optional[List[str]] = None):
272    auto_gen(repo_root_dir, root_dir, docs_dir, modules_to_ignore)
273    gather(repo_root_dir, root_dir, docs_dir, modules_to_ignore)
274    merge_docs(repo_root_dir, root_dir, docs_dir)