from pathlib import Path import os import sqlite3 import csv import re import shutil from datetime import datetime # ============================================================ # CONFIGURATION # ============================================================ IMPORT_FOLDER = Path("00-IMPORT") # All generated export artifacts live under one folder. EXPORT_FOLDER = Path("01-EXPORT") LOG_FOLDER = EXPORT_FOLDER / "00-LOG" SQLITE_DB = EXPORT_FOLDER / "00-EXPORT.db" SQL_DUMP_FILE = EXPORT_FOLDER / "00-EXPORT.sql" MD_EXPORT_FILE = EXPORT_FOLDER / "00-EXPORT.md" T4_EXPORT_FILE = EXPORT_FOLDER / "00-T4-EXPORT.md" T4_TWO_COLUMN_DB = EXPORT_FOLDER / "00-T4-EXPORT.db" T4_TWO_COLUMN_CSV = EXPORT_FOLDER / "00-T4-EXPORT.csv" FULL_EXPORT_CSV = EXPORT_FOLDER / "00-EXPORT.csv" # Convenience copies also written to the project root every run. ROOT_SQLITE_DB = Path("00-EXPORT.db") ROOT_EXPORT_CSV = Path("00-EXPORT.csv") ROOT_MASTER_LOG = Path("MASTER.log") ROOT_EXPORT_CSV_LOG = Path("00-EXPORT-CSV.log") ROOT_SQL_DUMP_LOG = Path("00-EXPORT-SQL.log") # All logs live in the dedicated log folder. LOG_FILE = LOG_FOLDER / "00-MD-IMPORT.log" T4_EXPORT_LOG_FILE = LOG_FOLDER / "00-T4-EXPORT.log" X_LOG_FILE = LOG_FOLDER / "00-X-TABLE-GOOGLE.log" MASTER_LOG_FILE = LOG_FOLDER / "MASTER.log" FULL_EXPORT_CSV_LOG_FILE = LOG_FOLDER / "00-EXPORT-CSV.log" SQL_DUMP_LOG_FILE = LOG_FOLDER / "00-EXPORT-SQL.log" TABLE_NAME = "AIO" DERECHO_TABLE_FOLDER = Path("00-Z-DERECHO-TABLE") X_IMPORT_FOLDER = Path("00-X-TABLE-FOLDER-IMPORT") X_TABLE_NAME = "X_TABLE_GOOGLE" X_GROUP_TABLE_PREFIX = "ZX-" X_MAIN_TABLE_NAME = "ZX-X-MAIN-AIO" SAMPLE_FILE_COUNT = 10 SAMPLE_PREFIX = "SAMPLE-" NAME_BLOCK_MARKER = "///" SPLIT_MARKER = "+++" EQUAL_FILENAME_MARKER = "=" TAG_OPEN_MARKER = "[" TAG_CLOSE_MARKER = "]" # Maximum number of characters stored in flattened T3. T3_MAX_CHARACTERS = 500 # Maximum number of characters stored in the Derecho flattened T3. # T3_FULL in the Derecho table remains unlimited. DERECHO_T3_MAX_CHARACTERS = 500 # ============================================================ # LOGGING / OUTPUT FOLDERS # ============================================================ LOG_LINES = [] X_LOG_LINES = [] def ensure_output_folders(): EXPORT_FOLDER.mkdir(parents=True, exist_ok=True) LOG_FOLDER.mkdir(parents=True, exist_ok=True) def recreate_log_files(): """Delete prior generated logs so every run starts with fresh log files.""" ensure_output_folders() for log_path in LOG_FOLDER.glob("*.log"): if log_path.is_file(): try: log_path.unlink() except OSError: pass # Root log files are convenience copies and are recreated later. for root_log in (ROOT_MASTER_LOG, ROOT_EXPORT_CSV_LOG, ROOT_SQL_DUMP_LOG): if root_log.exists() and root_log.is_file(): try: root_log.unlink() except OSError: pass def build_master_log(): """Concatenate every saved *.log into MASTER.log, excluding MASTER.log itself.""" ensure_output_folders() log_files = sorted( [ path for path in LOG_FOLDER.glob("*.log") if path.is_file() and path.name.lower() != MASTER_LOG_FILE.name.lower() ], key=lambda path: (path.stat().st_mtime, path.name.lower()) ) master_lines = [ "=" * 70, "MASTER LOG", "=" * 70, f"Created: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", f"Log folder: {LOG_FOLDER.resolve()}", f"Log files included: {len(log_files)}", "=" * 70, "", ] for log_path in log_files: master_lines.extend([ "#" * 70, f"SOURCE LOG: {log_path.name}", "#" * 70, ]) try: content = log_path.read_text(encoding="utf-8", errors="replace") except OSError as exc: content = f"[READ ERROR] {exc}\n" master_lines.append(content.rstrip("\n")) master_lines.extend(["", ""]) MASTER_LOG_FILE.write_text( "\n".join(master_lines).rstrip() + "\n", encoding="utf-8" ) return len(log_files) def write_export_log(log_path, title, details): """Recreate one dedicated export log with a compact run summary.""" ensure_output_folders() lines = [ "=" * 70, title, "=" * 70, f"Created: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", ] lines.extend(str(item) for item in details) lines.append("=" * 70) log_path.write_text("\n".join(lines) + "\n", encoding="utf-8") def reset_x_log_file(): ensure_output_folders() X_LOG_LINES.clear() X_LOG_FILE.write_text("", encoding="utf-8") def x_log(message=""): message = str(message) X_LOG_LINES.append(message) with X_LOG_FILE.open("a", encoding="utf-8") as log_handle: log_handle.write(message + "\n") def reset_log_file(): ensure_output_folders() # Create the current run log immediately so a log still exists # even if the script later stops with an error. LOG_FILE.write_text("", encoding="utf-8") def log(message=""): message = str(message) print(message) LOG_LINES.append(message) # Write every log entry immediately. This preserves the # diagnostic log if an exception happens before main() ends. with LOG_FILE.open("a", encoding="utf-8") as log_handle: log_handle.write(message + "\n") def write_log(): # Final rewrite guarantees the disk log exactly matches # LOG_LINES and avoids duplicate lines from repeated calls. LOG_FILE.write_text( "\n".join(LOG_LINES) + ("\n" if LOG_LINES else ""), encoding="utf-8" ) # ============================================================ # CREATE IMPORT ROOT # ============================================================ def create_import_folder(): IMPORT_FOLDER.mkdir( parents=True, exist_ok=True ) DERECHO_TABLE_FOLDER.mkdir( parents=True, exist_ok=True ) X_IMPORT_FOLDER.mkdir( parents=True, exist_ok=True ) log("=" * 70) log("IMPORT FOLDER") log("=" * 70) log( f"Import folder: " f"{IMPORT_FOLDER.resolve()}" ) log( f"Derecho table folder: " f"{DERECHO_TABLE_FOLDER.resolve()}" ) log( f"X import folder: " f"{X_IMPORT_FOLDER.resolve()}" ) log("") # ============================================================ # FIND ALL FOLDERS # ============================================================ def find_all_folders(): folders = [] # MAIN 00-IMPORT folders. The special X/Derecho roots now live at project root. folders.append((IMPORT_FOLDER, IMPORT_FOLDER)) for root, dirnames, _filenames in os.walk(IMPORT_FOLDER): root_path = Path(root) # Special folders are no longer nested here; keep this safeguard harmlessly in place. dirnames[:] = sorted( [ name for name in dirnames if (root_path / name) not in { X_IMPORT_FOLDER, DERECHO_TABLE_FOLDER, } ], key=str.casefold ) if root_path != IMPORT_FOLDER: folders.append((root_path, IMPORT_FOLDER)) # X import keeps its own sample-file tree, exactly as before. folders.append((X_IMPORT_FOLDER, X_IMPORT_FOLDER)) nested_x_folders = sorted( [ path for path in X_IMPORT_FOLDER.rglob("*") if path.is_dir() ], key=lambda path: str(path).lower() ) folders.extend( (folder, X_IMPORT_FOLDER) for folder in nested_x_folders ) return folders # ============================================================ # MAINTAIN EMPTY SAMPLE FILES IN EVERY 00-IMPORT / 00-X-IMPORT FOLDER # ============================================================ def create_sample_files(): log("=" * 70) log("SAMPLE FILE CREATION") log("=" * 70) folders = find_all_folders() total_created = 0 total_existing = 0 total_used = 0 total_empty = 0 log( f"Folders found: " f"{len(folders)}" ) log( f"Required EMPTY samples per folder: " f"{SAMPLE_FILE_COUNT}" ) log("") for folder, source_root in folders: relative_folder = ( folder.relative_to( source_root ) ) if str(relative_folder) == ".": folder_display = source_root.name else: folder_display = ( f"{source_root.name}/" f"{relative_folder.as_posix()}" ) log("-" * 70) log( f"FOLDER: " f"{folder_display}" ) log("-" * 70) # ---------------------------------------------------- # FIND ALL EXISTING SAMPLE-###.md FILES # ---------------------------------------------------- existing_samples = [] for path in folder.iterdir(): if not path.is_file(): continue name = path.name if not name.lower().startswith( SAMPLE_PREFIX.lower() ): continue if path.suffix.lower() != ".md": continue number_text = ( path.stem[ len(SAMPLE_PREFIX): ] ) if not number_text.isdigit(): continue number = int(number_text) existing_samples.append( (number, path) ) existing_samples.sort( key=lambda item: item[0] ) # ---------------------------------------------------- # COUNT EMPTY VS USED SAMPLE FILES # ---------------------------------------------------- empty_samples = [] used_samples = [] for number, sample_file in existing_samples: try: file_size = ( sample_file.stat().st_size ) except OSError as exc: log( f"STAT ERROR : " f"{sample_file} | {exc}" ) continue if file_size == 0: empty_samples.append( (number, sample_file) ) log( f"EMPTY : " f"{sample_file}" ) else: used_samples.append( (number, sample_file) ) log( f"USED : " f"{sample_file} " f"| SIZE={file_size}" ) empty_count = len( empty_samples ) used_count = len( used_samples ) existing_count = len( existing_samples ) total_existing += existing_count total_used += used_count total_empty += empty_count # ---------------------------------------------------- # FIND HIGHEST EXISTING SAMPLE NUMBER # ---------------------------------------------------- if existing_samples: highest_number = max( number for number, _ in existing_samples ) else: highest_number = 0 # ---------------------------------------------------- # KEEP REQUIRED EMPTY SAMPLE FILES AVAILABLE # ---------------------------------------------------- needed = max( 0, SAMPLE_FILE_COUNT - empty_count ) next_number = ( highest_number + 1 ) folder_created = 0 while folder_created < needed: sample_file = ( folder / f"{SAMPLE_PREFIX}" f"{next_number:03d}.md" ) # ------------------------------------------------ # NEVER OVERWRITE AN EXISTING SAMPLE FILE # ------------------------------------------------ if sample_file.exists(): next_number += 1 continue try: # Exclusive creation. If the file already exists, # Python raises FileExistsError instead of replacing it. with sample_file.open( "x", encoding="utf-8" ): pass folder_created += 1 total_created += 1 log( f"CREATED : " f"{sample_file}" ) except FileExistsError: pass except Exception as exc: log( f"CREATE ERROR : " f"{sample_file} | {exc}" ) next_number += 1 final_empty_count = ( empty_count + folder_created ) log("") log( f"Existing SAMPLE files : " f"{existing_count}" ) log( f"Used SAMPLE files : " f"{used_count}" ) log( f"Empty before creation : " f"{empty_count}" ) log( f"Created in folder : " f"{folder_created}" ) log( f"Final empty samples : " f"{final_empty_count}" ) if final_empty_count >= SAMPLE_FILE_COUNT: log( "STATUS : " f"OK - {SAMPLE_FILE_COUNT} EMPTY SAMPLE FILES AVAILABLE" ) else: log( "STATUS : " f"ERROR - LESS THAN {SAMPLE_FILE_COUNT} EMPTY SAMPLE FILES" ) log("") log("=" * 70) log("SAMPLE CREATION TOTAL") log("=" * 70) log( f"Folders processed : " f"{len(folders)}" ) log( f"Existing SAMPLE files : " f"{total_existing}" ) log( f"Used SAMPLE files : " f"{total_used}" ) log( f"Empty samples found : " f"{total_empty}" ) log( f"New samples created : " f"{total_created}" ) log("") # ============================================================ # GLOBAL DENSECODE LETTER GENERATOR # ============================================================ def number_to_letters(number): """ 1 -> a 2 -> b ... 26 -> z 27 -> aa """ result = "" while number > 0: number -= 1 result = ( chr( ord("a") + (number % 26) ) + result ) number //= 26 return result # ============================================================ # GLOBAL DENSECODE # ============================================================ def make_densecode(row_number): """ a+0001 ... a+9999 b+0001 ... """ if row_number < 1: raise ValueError( "row_number must be >= 1" ) group_number = ( ((row_number - 1) // 9999) + 1 ) numeric_number = ( ((row_number - 1) % 9999) + 1 ) letters = number_to_letters( group_number ) return ( f"{letters}" f"+" f"{numeric_number:04d}" ) # ============================================================ # DYNAMIC DENSECODE PREFIX # ============================================================ def number_to_dynamic_letters(number): """ 1 -> A 2 -> B ... 26 -> Z 27 -> AA ... 702 -> ZZ 703 -> AAA ... 18278 -> ZZZ """ if number < 1: raise ValueError( "Dynamic MD number must be >= 1" ) if number > 18278: raise ValueError( "DYNAMIC_DENSECODE exceeded ZZZ." ) result = "" while number > 0: number -= 1 result = ( chr( ord("A") + (number % 26) ) + result ) number //= 26 return result # ============================================================ # CREATE DYNAMIC DENSECODE # ============================================================ def make_dynamic_densecode( file_number, row_number ): if row_number < 1: raise ValueError( "Dynamic row number must be >= 1" ) prefix = number_to_dynamic_letters( file_number ) return ( f"{prefix}" f"+" f"{row_number:04d}" ) # ============================================================ # RECREATE DATABASE # ============================================================ def recreate_database(): log("=" * 70) log("DATABASE") log("=" * 70) if SQLITE_DB.exists(): SQLITE_DB.unlink() log( f"Deleted old database: " f"{SQLITE_DB.resolve()}" ) if SQL_DUMP_FILE.exists(): SQL_DUMP_FILE.unlink() log( f"Deleted old SQL dump: " f"{SQL_DUMP_FILE.resolve()}" ) if MD_EXPORT_FILE.exists(): MD_EXPORT_FILE.unlink() log( f"Deleted old MD export: " f"{MD_EXPORT_FILE.resolve()}" ) conn = sqlite3.connect( SQLITE_DB ) cursor = conn.cursor() # ======================================================== # DATABASE COLUMN ORDER # # DYNAMIC_DENSECODE REMAINS LAST # ======================================================== # IMPORTANT: Create AIO FIRST. SQLite preserves this creation order in # sqlite_schema/sqlite_master when no client-side alphabetical sort is applied. cursor.execute( f''' CREATE TABLE "{TABLE_NAME}" ( ID INTEGER PRIMARY KEY AUTOINCREMENT, DENSECODE TEXT NOT NULL UNIQUE, TAGS TEXT, T2 TEXT, TEXT_TAG TEXT, IMPORT_SOURCE TEXT, RELATIVE_PATH TEXT, T1 TEXT, T3 TEXT, DYNAMIC_DENSECODE TEXT NOT NULL UNIQUE, T4 TEXT ) ''' ) cursor.execute( f''' CREATE TABLE "{X_TABLE_NAME}" ( ID INTEGER PRIMARY KEY AUTOINCREMENT, DENSECODE TEXT NOT NULL UNIQUE, TAGS TEXT, T2 TEXT, TEXT_TAG TEXT, RELATIVE_PATH TEXT, T1 TEXT, T3 TEXT, DYNAMIC_DENSECODE TEXT NOT NULL UNIQUE, T4 TEXT ) ''' ) conn.commit() log( f"Created database: " f"{SQLITE_DB.resolve()}" ) log( f"Created table: " f"{TABLE_NAME}" ) log("") log("Columns:") log(" ID") log(" DENSECODE") log(" TAGS") log(" T2") log(" TEXT_TAG") log(" IMPORT_SOURCE") log(" RELATIVE_PATH") log(" T1") log(" T3") log(" DYNAMIC_DENSECODE") log(" T4") log("") log(f"Separate X table: {X_TABLE_NAME}") log(" ID") log(" DENSECODE") log(" TAGS") log(" T2") log(" TEXT_TAG") log(" RELATIVE_PATH") log(" T1") log(" T3") log(" DYNAMIC_DENSECODE") log(" T4") log("") return conn # ============================================================ # FIND ALL *.MD RECURSIVELY # ============================================================ def find_markdown_files(): # Use os.walk so every directory under 00-IMPORT is visited. # Extension comparison is case-insensitive, so .md, .MD, .Md, # and .mD are all processed. markdown_files = [] folders_scanned = 0 for root, dirnames, filenames in os.walk(IMPORT_FOLDER): root_path = Path(root) # X/Derecho now live at project root. This exclusion remains as a safeguard # if legacy nested folders happen to still exist under 00-IMPORT. dirnames[:] = sorted( [ name for name in dirnames if (root_path / name) not in { X_IMPORT_FOLDER, DERECHO_TABLE_FOLDER, } ], key=str.casefold ) filenames.sort(key=str.casefold) folders_scanned += 1 for filename in filenames: file_path = root_path / filename if file_path.suffix.casefold() == ".md": markdown_files.append(file_path) markdown_files.sort( key=lambda path: ( path.relative_to(IMPORT_FOLDER) .as_posix() .casefold() ) ) log( f"Recursive MD scan folders : " f"{folders_scanned}" ) log( f"Recursive *.md files found: " f"{len(markdown_files)}" ) log("") return markdown_files # ============================================================ # PARSE TAG LINE # ============================================================ def parse_tag_line(line): """ Example: [computer,yardbirds,mall] becomes: [ "computer", "yardbirds", "mall" ] Empty tags are ignored. Spaces around each tag are stripped. """ stripped = line.strip() if not ( stripped.startswith( TAG_OPEN_MARKER ) and stripped.endswith( TAG_CLOSE_MARKER ) ): return None inner = stripped[1:-1] tags = [] for item in inner.split(","): tag = item.strip() if tag: tags.append( tag ) return tags # ============================================================ # NORMALIZE TAGS # ============================================================ def normalize_tags(tags): """ Removes duplicate tags while preserving their original order. Example: computer,yardbirds,computer,mall becomes: computer,yardbirds,mall """ result = [] seen = set() for tag in tags: key = tag.casefold() if key in seen: continue seen.add( key ) result.append( tag ) return result # ============================================================ # CONVERT T2 CONTENT TO UNIQUE TEXT TAGS # ============================================================ def content_to_text_tags(content): """Convert T2 content into comma-delimited unique tags, excluding web links.""" text = str(content or "") # Remove complete HTTP/HTTPS URLs before tokenizing so URL pieces # such as https, domain names, paths, and query strings do not # become TEXT_TAG values. text = re.sub( r"https?://\S+", " ", text, flags=re.IGNORECASE ) tokens = re.findall( r"[^\W_]+(?:['’\-][^\W_]+)*", text, flags=re.UNICODE ) result = [] seen = set() for token in tokens: key = token.casefold() if key in seen: continue seen.add(key) result.append(token) return ",".join(result) # ============================================================ # DERECHO SINGLE-TABLE SUPPORT # ============================================================ def quote_identifier(name): return '"' + str(name).replace('"', '""') + '"' def get_derecho_table_name(file_path=None): # All Markdown files under the Derecho root share one table. return DERECHO_TABLE_FOLDER.name def ensure_derecho_table(cursor, table_name): cursor.execute( f''' CREATE TABLE IF NOT EXISTS {quote_identifier(table_name)} ( ID INTEGER PRIMARY KEY AUTOINCREMENT, DYNAMIC_DENSECODE TEXT NOT NULL UNIQUE, TAGS TEXT, T3 TEXT, T3_FULL TEXT, RELATIVE_PATH TEXT, DENSECODE TEXT NOT NULL UNIQUE ) ''' ) # ============================================================ # INSERT ROW # ============================================================ def insert_row( cursor, densecode_counter, dynamic_file_number, dynamic_row_counter, t1, t2, relative_path, import_source, tags, section_number, row_type ): # -------------------------------------------------------- # GLOBAL DENSECODE # -------------------------------------------------------- densecode_counter += 1 densecode = make_densecode( densecode_counter ) # -------------------------------------------------------- # DYNAMIC DENSECODE # -------------------------------------------------------- dynamic_row_counter += 1 dynamic_densecode = ( make_dynamic_densecode( dynamic_file_number, dynamic_row_counter ) ) # -------------------------------------------------------- # RELATIVE PATH # -------------------------------------------------------- relative_path_string = ( relative_path.as_posix() ) # -------------------------------------------------------- # TAG STRING # -------------------------------------------------------- tags_string = ",".join( tags ) # TEXT_TAG contains unique tokens extracted from T2 plus a copy of TAGS. # Duplicate matching is case-insensitive and preserves first occurrence. text_tag_items = [] text_tag_seen = set() for item in content_to_text_tags(t2).split(","): item = item.strip() if not item: continue key = item.casefold() if key in text_tag_seen: continue text_tag_seen.add(key) text_tag_items.append(item) for tag in tags: tag = str(tag).strip() if not tag: continue key = tag.casefold() if key in text_tag_seen: continue text_tag_seen.add(key) text_tag_items.append(tag) text_tag = ",".join(text_tag_items) # -------------------------------------------------------- # T3_FULL = FULL SINGLE-LINE COPY OF T2 - NO LIMIT # T3 = LIMITED SINGLE-LINE COPY USED BY CSV / MD # -------------------------------------------------------- t3_full = " ".join( str(t2 or "").split() ) t3 = t3_full[:T3_MAX_CHARACTERS] # -------------------------------------------------------- # T4 = DYNAMIC_DENSECODE + " | " + T3 # -------------------------------------------------------- t4 = f"{dynamic_densecode} | {t3}" # ======================================================== # INSERT # # DYNAMIC_DENSECODE LAST # ======================================================== cursor.execute( f''' INSERT INTO "{TABLE_NAME}" ( DENSECODE, TAGS, T2, TEXT_TAG, IMPORT_SOURCE, RELATIVE_PATH, T1, T3, DYNAMIC_DENSECODE, T4 ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ''', ( densecode, tags_string, t2, text_tag, import_source, relative_path_string, t1, t3, dynamic_densecode, t4 ) ) log( f"{row_type} | " f"DENSECODE={densecode} | " f"SECTION={section_number} | " f"T1={t1!r} | " f"T3_PREVIEW_100={log_value_preview(t3, 100)!r} | " f"T3_FULL_LENGTH={len(t3_full)} | " f"T4_PREVIEW_100={log_value_preview(t4, 100)!r} | " f"T2_PREVIEW_50={log_t2_preview(t2)!r} | " f"T2_LENGTH={len(t2)} | " f"TEXT_TAG_FIRST_10={log_first_text_tags(text_tag)!r} | " f"RELATIVE_PATH=" f"{relative_path_string!r} | " f"TAGS={tags_string!r} | " f"DYNAMIC={dynamic_densecode}" ) return ( densecode_counter, dynamic_row_counter ) # ============================================================ # SPLIT FILE AT +++ # ============================================================ def split_into_sections( text, relative_path ): """ +++ creates independent sections. +++ itself is not included in T2. """ lines = text.splitlines() sections = [] current_lines = [] split_count = 0 for line_number, raw_line in enumerate( lines, start=1 ): stripped = raw_line.strip() if stripped == SPLIT_MARKER: split_count += 1 sections.append( current_lines ) log( f"+++ SPLIT | " f"{relative_path.as_posix()} | " f"LINE={line_number} | " f"SECTION CLOSED=" f"{len(sections)}" ) current_lines = [] continue current_lines.append( raw_line ) sections.append( current_lines ) return ( sections, split_count ) # ============================================================ # PARSE ONE +++ SECTION # ============================================================ def parse_section( section_lines, relative_path, section_number ): """ SUPPORTED METADATA: 1. Filename aliases: /// NAME ONE NAME TWO /// 2. Single-line filename aliases: =NAME THREE 3. Tags: [computer,yardbirds,mall] Everything else becomes T2. """ filenames = [] tags = [] content_lines = [] inside_name_block = False name_block_count = 0 equal_filename_count = 0 tag_line_count = 0 for local_line_number, raw_line in enumerate( section_lines, start=1 ): stripped = raw_line.strip() # ==================================================== # /// OPEN/CLOSE # ==================================================== if stripped == NAME_BLOCK_MARKER: if inside_name_block: inside_name_block = False log( f"/// CLOSE | " f"{relative_path.as_posix()} | " f"SECTION={section_number} | " f"LINE={local_line_number}" ) else: inside_name_block = True name_block_count += 1 log( f"/// OPEN | " f"{relative_path.as_posix()} | " f"SECTION={section_number} | " f"LINE={local_line_number}" ) continue # ==================================================== # INSIDE /// = FILENAME # ==================================================== if inside_name_block: if stripped: filenames.append( { "name": stripped, "source": "///" } ) log( f"/// FILENAME FOUND | " f"{relative_path.as_posix()} | " f"SECTION={section_number} | " f"LINE={local_line_number} | " f"T1={stripped!r}" ) continue # ==================================================== # =* FILENAME # ==================================================== if ( stripped and stripped.startswith( EQUAL_FILENAME_MARKER ) ): filenames.append( { "name": stripped, "source": "=" } ) equal_filename_count += 1 log( f"= FILENAME FOUND | " f"{relative_path.as_posix()} | " f"SECTION={section_number} | " f"LINE={local_line_number} | " f"T1={stripped!r}" ) continue # ==================================================== # [*] TAG MARKER # # EXAMPLE: # # [computer,yardbirds,mall] # # = 3 TAGS # ==================================================== parsed_tags = parse_tag_line( stripped ) if parsed_tags is not None: tag_line_count += 1 for tag in parsed_tags: tags.append( tag ) log( f"TAG FOUND | " f"{relative_path.as_posix()} | " f"SECTION={section_number} | " f"LINE={local_line_number} | " f"TAG={tag!r}" ) log( f"TAG LINE | " f"{relative_path.as_posix()} | " f"SECTION={section_number} | " f"LINE={local_line_number} | " f"COUNT={len(parsed_tags)} | " f"RAW={stripped!r}" ) # Tag line does not go into T2 continue # ==================================================== # NORMAL CONTENT # ==================================================== content_lines.append( raw_line ) # -------------------------------------------------------- # REMOVE DUPLICATE TAGS # -------------------------------------------------------- tags = normalize_tags( tags ) # -------------------------------------------------------- # T2 # -------------------------------------------------------- content = ( "\n".join(content_lines) .strip() ) return { "filenames": filenames, "content": content, "tags": tags, "name_blocks": name_block_count, "equal_filenames": equal_filename_count, "tag_lines": tag_line_count } # ============================================================ # IMPORT ONE *.MD # ============================================================ def import_md_file( conn, file_path, densecode_counter, dynamic_file_number, source_root=IMPORT_FOLDER ): cursor = conn.cursor() relative_path = ( file_path.relative_to( source_root ) ) # Main AIO source marker. if source_root == X_IMPORT_FOLDER: if len(relative_path.parts) > 1: import_source = f"00-X-IMPORT/{relative_path.parts[0]}" else: import_source = "00-X-IMPORT" else: import_source = "MAIN" log("-" * 70) log( f"FILE: " f"{relative_path.as_posix()}" ) log("-" * 70) # -------------------------------------------------------- # READ FILE # -------------------------------------------------------- try: text = file_path.read_text( encoding="utf-8-sig" ) except UnicodeDecodeError: text = file_path.read_text( encoding="utf-8", errors="replace" ) # -------------------------------------------------------- # EMPTY FILE # -------------------------------------------------------- if not text.strip(): log( f"EMPTY FILE SKIPPED | " f"{relative_path.as_posix()}" ) log("") return ( densecode_counter, 0, # imported_rows 0, # filename_count 0, # split_count 0, # name_block_count 0, # equal_filename_count 0, # tag_line_count 0, # tag_count 0 # real_sections ) # -------------------------------------------------------- # DYNAMIC PREFIX # -------------------------------------------------------- dynamic_prefix = ( number_to_dynamic_letters( dynamic_file_number ) ) log( f"DYNAMIC FILE PREFIX | " f"{relative_path.as_posix()} | " f"PREFIX={dynamic_prefix}" ) dynamic_row_counter = 0 # ======================================================== # SPLIT AT +++ # ======================================================== ( sections, split_count ) = split_into_sections( text, relative_path ) imported_rows = 0 total_filenames = 0 total_name_blocks = 0 total_equal_filenames = 0 total_tag_lines = 0 total_tags = 0 real_sections = 0 default_name = ( file_path.stem ) # ======================================================== # PROCESS EACH SECTION # ======================================================== for section_number, section_lines in enumerate( sections, start=1 ): if not any( line.strip() for line in section_lines ): log( f"EMPTY SECTION SKIPPED | " f"{relative_path.as_posix()} | " f"SECTION={section_number}" ) continue parsed = parse_section( section_lines, relative_path, section_number ) filenames = ( parsed["filenames"] ) content = ( parsed["content"] ) tags = ( parsed["tags"] ) name_blocks = ( parsed["name_blocks"] ) equal_filenames = ( parsed["equal_filenames"] ) tag_lines = ( parsed["tag_lines"] ) total_name_blocks += ( name_blocks ) total_equal_filenames += ( equal_filenames ) total_filenames += ( len(filenames) ) total_tag_lines += ( tag_lines ) total_tags += ( len(tags) ) # ---------------------------------------------------- # NOTHING USABLE # # TAGS ALONE ARE ALSO CONSIDERED DATA. # ---------------------------------------------------- if ( not filenames and not content and not tags ): log( f"EMPTY PARSED SECTION SKIPPED | " f"{relative_path.as_posix()} | " f"SECTION={section_number}" ) continue real_sections += 1 # ==================================================== # ALIASES # ==================================================== if filenames: for filename_info in filenames: filename = ( filename_info["name"] ) filename_source = ( filename_info["source"] ) if filename_source == "=": row_type = ( "= FILENAME" ) else: row_type = ( "/// FILENAME" ) ( densecode_counter, dynamic_row_counter ) = insert_row( cursor=cursor, densecode_counter=densecode_counter, dynamic_file_number=dynamic_file_number, dynamic_row_counter=dynamic_row_counter, t1=filename, t2=content, relative_path=relative_path, import_source=import_source, tags=tags, section_number=section_number, row_type=row_type ) imported_rows += 1 # ==================================================== # NO FILENAME ALIAS # # USE ACTUAL MD FILENAME # ==================================================== else: ( densecode_counter, dynamic_row_counter ) = insert_row( cursor=cursor, densecode_counter=densecode_counter, dynamic_file_number=dynamic_file_number, dynamic_row_counter=dynamic_row_counter, t1=default_name, t2=content, relative_path=relative_path, import_source=import_source, tags=tags, section_number=section_number, row_type="DEFAULT-FILENAME" ) imported_rows += 1 conn.commit() # -------------------------------------------------------- # FILE SUMMARY # -------------------------------------------------------- log("") log( f"Dynamic prefix : " f"{dynamic_prefix}" ) log( f"Dynamic rows : " f"{dynamic_row_counter}" ) log( f"+++ splits found : " f"{split_count}" ) log( f"Non-empty sections : " f"{real_sections}" ) log( f"Filename aliases total : " f"{total_filenames}" ) log( f"/// blocks : " f"{total_name_blocks}" ) log( f"= filenames found : " f"{total_equal_filenames}" ) log( f"Tag lines found : " f"{total_tag_lines}" ) log( f"Tags found : " f"{total_tags}" ) log( f"Rows imported : " f"{imported_rows}" ) log( f"Relative path : " f"{relative_path.as_posix()}" ) log("") return ( densecode_counter, imported_rows, total_filenames, split_count, total_name_blocks, total_equal_filenames, total_tag_lines, total_tags, real_sections ) # ============================================================ # FIND DERECHO *.MD RECURSIVELY # ============================================================ def find_derecho_markdown_files(): files = [] for root, dirnames, filenames in os.walk(DERECHO_TABLE_FOLDER): dirnames.sort(key=str.casefold) filenames.sort(key=str.casefold) root_path = Path(root) for filename in filenames: file_path = root_path / filename if file_path.suffix.casefold() == ".md": files.append(file_path) files.sort( key=lambda path: path.relative_to(DERECHO_TABLE_FOLDER) .as_posix().casefold() ) return files # ============================================================ # IMPORT ALL DERECHO FILES INTO ONE TABLE ONLY # ============================================================ def import_derecho_tables(conn): md_files = find_derecho_markdown_files() cursor = conn.cursor() log("=" * 70) log("DERECHO SINGLE-TABLE IMPORT") log("=" * 70) log(f"Derecho *.md files found recursively: {len(md_files)}") log("These files DO NOT enter AIO.") log("") derecho_densecode_counter = 0 derecho_dynamic_file_counter = 0 total_rows = 0 table_name = get_derecho_table_name() ensure_derecho_table(cursor, table_name) for file_path in md_files: relative_path = file_path.relative_to(DERECHO_TABLE_FOLDER) try: text = file_path.read_text(encoding="utf-8-sig") except UnicodeDecodeError: text = file_path.read_text(encoding="utf-8", errors="replace") if not text.strip(): log(f"DERECHO EMPTY FILE SKIPPED | {relative_path.as_posix()}") continue derecho_dynamic_file_counter += 1 dynamic_row_counter = 0 log("-" * 70) log(f"DERECHO FILE : {relative_path.as_posix()}") log(f"TABLE : {table_name}") log("-" * 70) sections, _ = split_into_sections(text, relative_path) for section_number, section_lines in enumerate(sections, start=1): if not any(line.strip() for line in section_lines): continue parsed = parse_section(section_lines, relative_path, section_number) content = parsed["content"] tags = parsed["tags"] filenames = parsed["filenames"] if not filenames and not content and not tags: continue row_count = len(filenames) if filenames else 1 for _ in range(row_count): derecho_densecode_counter += 1 dynamic_row_counter += 1 densecode = make_densecode(derecho_densecode_counter) dynamic_densecode = make_dynamic_densecode( derecho_dynamic_file_counter, dynamic_row_counter ) tags_string = ",".join(tags) t3_full = " ".join(str(content or "").split()) t3 = t3_full[:DERECHO_T3_MAX_CHARACTERS] relative_path_string = relative_path.as_posix() cursor.execute( f''' INSERT INTO {quote_identifier(table_name)} (DYNAMIC_DENSECODE, TAGS, T3, T3_FULL, RELATIVE_PATH, DENSECODE) VALUES (?, ?, ?, ?, ?, ?) ''', ( dynamic_densecode, tags_string, t3, t3_full, relative_path_string, densecode ) ) total_rows += 1 conn.commit() log("") log(f"Derecho table : {table_name}") log("Derecho tables created : 1") log(f"Derecho rows imported : {total_rows}") log(f"Derecho T3 limit : {DERECHO_T3_MAX_CHARACTERS} characters") log("Derecho T3_FULL : NO CHARACTER LIMIT") log("Derecho RELATIVE_PATH : ENABLED") log("") return { "files": len(md_files), "tables": 1, "rows": total_rows } # ============================================================ # LOG PREVIEW HELPERS # ============================================================ def log_value_preview(value, max_chars): """Return a single-line preview limited to max_chars for logs only.""" return " ".join(str(value or "").split())[:max_chars] def log_t2_preview(value, max_chars=50): """Return a single-line T2 preview limited to max_chars for logs only.""" return log_value_preview(value, max_chars) def log_first_text_tags(value, max_tags=10): """Return only the first max_tags comma-delimited TEXT_TAG items for logs.""" items = [item.strip() for item in str(value or "").split(",") if item.strip()] return ",".join(items[:max_tags]) # ============================================================ # X IMPORT GROUP TABLE SUPPORT # ============================================================ def get_x_group_table_name(file_path): """ Loose files directly inside 00-X-TABLE-FOLDER-IMPORT -> ZX-X-MAIN-AIO. Files inside a first-level folder use a ZX- prefixed table name. Example: 00-X-TABLE-FOLDER-IMPORT/CARS/*.md -> ZX-CARS. Any deeper nested folders remain part of that same first-level table. """ relative_path = file_path.relative_to(X_IMPORT_FOLDER) parts = relative_path.parts if len(parts) <= 1: return X_MAIN_TABLE_NAME return f"{X_GROUP_TABLE_PREFIX}{parts[0]}" def ensure_x_group_table(cursor, table_name): """Create one X group table with the same row layout as X_TABLE_GOOGLE.""" cursor.execute( f''' CREATE TABLE IF NOT EXISTS {quote_identifier(table_name)} ( ID INTEGER PRIMARY KEY AUTOINCREMENT, DENSECODE TEXT NOT NULL UNIQUE, TAGS TEXT, T2 TEXT, TEXT_TAG TEXT, RELATIVE_PATH TEXT, T1 TEXT, T3 TEXT, DYNAMIC_DENSECODE TEXT NOT NULL UNIQUE, T4 TEXT ) ''' ) def get_x_qualified_folder_table_name(file_path): """Return an exact root-qualified table name for each X folder.""" parent = file_path.parent if parent == X_IMPORT_FOLDER: return f"{X_IMPORT_FOLDER.name}/00-AIO" relative_parent = parent.relative_to(X_IMPORT_FOLDER) return X_IMPORT_FOLDER.name + "/" + "/".join(relative_parent.parts) def copy_aio_rows_to_x_qualified_table(conn, table_name, first_id, last_id): """Copy one X file's AIO rows into its exact-folder X table.""" if not table_name or last_id < first_id: return 0 cursor = conn.cursor() ensure_x_group_table(cursor, table_name) cursor.execute( f''' SELECT DENSECODE, TAGS, T2, TEXT_TAG, RELATIVE_PATH, T1, T3, DYNAMIC_DENSECODE, T4 FROM {quote_identifier(TABLE_NAME)} WHERE ID BETWEEN ? AND ? ORDER BY ID ''', (first_id, last_id) ) rows = cursor.fetchall() if rows: cursor.executemany( f''' INSERT INTO {quote_identifier(table_name)} (DENSECODE, TAGS, T2, TEXT_TAG, RELATIVE_PATH, T1, T3, DYNAMIC_DENSECODE, T4) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ''', rows ) conn.commit() return len(rows) def copy_aio_rows_to_named_x_table(conn, table_name, first_id, last_id): """Copy newly imported AIO rows into one folder-specific X table.""" if last_id < first_id: return 0 cursor = conn.cursor() ensure_x_group_table(cursor, table_name) cursor.execute( f''' SELECT DENSECODE, TAGS, T2, TEXT_TAG, RELATIVE_PATH, T1, T3, DYNAMIC_DENSECODE, T4 FROM {quote_identifier(TABLE_NAME)} WHERE ID BETWEEN ? AND ? ORDER BY ID ''', (first_id, last_id) ) rows = cursor.fetchall() cursor.executemany( f''' INSERT INTO {quote_identifier(table_name)} ( DENSECODE, TAGS, T2, TEXT_TAG, RELATIVE_PATH, T1, T3, DYNAMIC_DENSECODE, T4 ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ''', rows ) conn.commit() return len(rows) # ============================================================ # REBUILD X_TABLE_GOOGLE FROM ALL ZX-* TABLES # ============================================================ def rebuild_x_table_from_zx_tables(conn): """ Rebuild X_TABLE_GOOGLE as the concatenation of every ZX-* table. IMPORTANT: - DENSECODE is copied exactly as stored in each ZX-* table. - DYNAMIC_DENSECODE is copied exactly as stored in each ZX-* table. - No densecode values are regenerated or renumbered. - The ZX-* rows were originally copied from AIO, so their original AIO-assigned codes are preserved in the master X table. """ cursor = conn.cursor() cursor.execute( "SELECT name FROM sqlite_master " "WHERE type='table' AND name LIKE ? ORDER BY name COLLATE NOCASE", (f"{X_GROUP_TABLE_PREFIX}%",) ) zx_tables = [row[0] for row in cursor.fetchall()] cursor.execute(f'DELETE FROM {quote_identifier(X_TABLE_NAME)}') total_rows = 0 for table_name in zx_tables: cursor.execute( f""" SELECT DENSECODE, TAGS, T2, TEXT_TAG, RELATIVE_PATH, T1, T3, DYNAMIC_DENSECODE, T4 FROM {quote_identifier(table_name)} ORDER BY ID """ ) rows = cursor.fetchall() if rows: cursor.executemany( f""" INSERT INTO {quote_identifier(X_TABLE_NAME)} ( DENSECODE, TAGS, T2, TEXT_TAG, RELATIVE_PATH, T1, T3, DYNAMIC_DENSECODE, T4 ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) """, rows ) total_rows += len(rows) conn.commit() return zx_tables, total_rows # ============================================================ # FIND 00-X-TABLE-FOLDER-IMPORT *.MD / *.ND RECURSIVELY # ============================================================ def find_x_import_files(): files = [] folders_scanned = 0 for root, dirnames, filenames in os.walk(X_IMPORT_FOLDER): dirnames.sort(key=str.casefold) filenames.sort(key=str.casefold) folders_scanned += 1 root_path = Path(root) for filename in filenames: file_path = root_path / filename # User requested *.ND. *.MD is also accepted so the X folder # can use the same Markdown workflow as 00-IMPORT. if file_path.suffix.casefold() in {".md", ".nd"}: files.append(file_path) files.sort( key=lambda path: path.relative_to(X_IMPORT_FOLDER) .as_posix().casefold() ) log(f"X recursive folders scanned : {folders_scanned}") log(f"X *.md/*.nd files found : {len(files)}") log("") return files # ============================================================ # COPY NEW AIO ROWS INTO X_TABLE_GOOGLE # ============================================================ def copy_aio_rows_to_x_table(conn, first_id, last_id): if last_id < first_id: return 0 cursor = conn.cursor() cursor.execute( f""" SELECT DENSECODE, TAGS, T2, TEXT_TAG, RELATIVE_PATH, T1, T3, DYNAMIC_DENSECODE, T4 FROM "{TABLE_NAME}" WHERE ID BETWEEN ? AND ? ORDER BY ID """, (first_id, last_id) ) rows = cursor.fetchall() cursor.executemany( f""" INSERT INTO "{X_TABLE_NAME}" ( DENSECODE, TAGS, T2, TEXT_TAG, RELATIVE_PATH, T1, T3, DYNAMIC_DENSECODE, T4 ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) """, rows ) conn.commit() return len(rows) # ============================================================ # IMPORT X FILES INTO AIO + EXACT FOLDER TABLE + X_TABLE_GOOGLE # ============================================================ def import_x_files(conn, densecode_counter, dynamic_file_counter): x_files = find_x_import_files() cursor = conn.cursor() log("=" * 70) log("X IMPORT -> AIO + EXACT FOLDER TABLE + X_TABLE_GOOGLE") log("=" * 70) log(f"X source folder : {X_IMPORT_FOLDER.resolve()}") log(f"X target table : {X_TABLE_NAME}") log("Legacy ZX-* duplicate tables: DISABLED") log("X files are NOT imported into 00-Z-DERECHO-TABLE.") log("") x_log("=" * 70) x_log("X_TABLE_GOOGLE IMPORT LOG") x_log("=" * 70) x_log(f"Started: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") x_log(f"Source folder: {X_IMPORT_FOLDER.resolve()}") x_log(f"Database: {SQLITE_DB.resolve()}") x_log(f"Table: {X_TABLE_NAME}") x_log("Targets: AIO + exact root-qualified folder table + X_TABLE_GOOGLE") x_log("Legacy ZX-* duplicate tables: DISABLED") x_log("Excluded target: 00-Z-DERECHO-TABLE") x_log("") total_rows = 0 x_table_rows = 0 dynamic_files = 0 processed_files = 0 folder_tables_used = {} for file_path in x_files: processed_files += 1 relative_path = file_path.relative_to(X_IMPORT_FOLDER) cursor.execute(f'SELECT COALESCE(MAX(ID), 0) FROM "{TABLE_NAME}"') before_max_id = cursor.fetchone()[0] proposed_dynamic_file_number = dynamic_file_counter + 1 ( densecode_counter, imported_rows, filename_count, split_count, name_block_count, equal_filename_count, tag_line_count, tag_count, real_sections ) = import_md_file( conn, file_path, densecode_counter, proposed_dynamic_file_number, source_root=X_IMPORT_FOLDER ) if imported_rows > 0: dynamic_file_counter += 1 dynamic_files += 1 cursor.execute(f'SELECT COALESCE(MAX(ID), 0) FROM "{TABLE_NAME}"') after_max_id = cursor.fetchone()[0] # One exact folder table only. No legacy ZX-* copy is created. x_qualified_table = get_x_qualified_folder_table_name(file_path) x_qualified_rows = copy_aio_rows_to_x_qualified_table( conn, x_qualified_table, before_max_id + 1, after_max_id ) if x_qualified_rows: folder_tables_used[x_qualified_table] = ( folder_tables_used.get(x_qualified_table, 0) + x_qualified_rows ) # X_TABLE_GOOGLE remains the all-in-one X master table, but it is # populated directly from this file's newly-created AIO rows. copied_rows = copy_aio_rows_to_x_table( conn, before_max_id + 1, after_max_id ) x_table_rows += copied_rows total_rows += imported_rows x_log( f"FILE={relative_path.as_posix()} | " f"ROWS_AIO={imported_rows} | " f"ROWS_X_TABLE={copied_rows} | " f"QUALIFIED_TABLE={x_qualified_table} | " f"ROWS_QUALIFIED_TABLE={x_qualified_rows} | " f"SPLITS={split_count} | " f"SECTIONS={real_sections} | " f"ALIASES={filename_count} | " f"TAGS={tag_count}" ) log(f"X files processed : {processed_files}") log(f"X files with DB rows : {dynamic_files}") log(f"X rows added to AIO : {total_rows}") log(f"X_TABLE_GOOGLE rows : {x_table_rows}") log("X_TABLE_GOOGLE source : DIRECT COPY OF EACH X FILE'S AIO ROWS") log("Legacy ZX-* tables : DISABLED - PREVENTS DOUBLE IMPORT TABLES") log("X densecodes preserved : YES - ORIGINAL AIO VALUES") if folder_tables_used: log("X exact folder tables:") for table_name in sorted(folder_tables_used, key=str.casefold): log(f" {table_name} | ROWS={folder_tables_used[table_name]}") log(f"X log file : {X_LOG_FILE.resolve()}") log("") x_log("") x_log(f"Files processed: {processed_files}") x_log(f"Files with DB rows: {dynamic_files}") x_log(f"Rows added to AIO: {total_rows}") x_log(f"Rows in {X_TABLE_NAME}: {x_table_rows}") x_log("Master build: DIRECT COPY FROM NEW AIO ROWS") x_log("Legacy ZX-* tables: DISABLED") x_log("DENSECODE preservation: ORIGINAL AIO VALUES - NO RENUMBERING") x_log("DYNAMIC_DENSECODE preservation: ORIGINAL AIO VALUES - NO RENUMBERING") if folder_tables_used: x_log("Exact folder tables:") for table_name in sorted(folder_tables_used, key=str.casefold): x_log(f" {table_name} | ROWS={folder_tables_used[table_name]}") x_log(f"Finished: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") x_log("=" * 70) return { "files": processed_files, "rows": total_rows, "dynamic_files": dynamic_files, "x_table_rows": x_table_rows, "densecode_counter": densecode_counter, "dynamic_file_counter": dynamic_file_counter } # ============================================================ # NORMAL 00-IMPORT FOLDER TABLE SUPPORT # ============================================================ def get_import_folder_table_name(file_path): """ Create a ROOT-QUALIFIED table name for each immediate parent folder. Root files: 00-IMPORT/file.md -> 0000-AIO First-level folder: 00-IMPORT/CBU/file.md -> 00-IMPORT/CBU Nested folders: 00-IMPORT/01-TECH CIVIL/DIBUJO/file.md -> 00-IMPORT/01-TECH CIVIL/DIBUJO 00-IMPORT/01-TECH CIVIL/DIBUJO/AUTOCAD/file.md -> 00-IMPORT/01-TECH CIVIL/DIBUJO/AUTOCAD Each level receives only files directly inside that folder. AIO remains unchanged. """ parent = file_path.parent if parent == IMPORT_FOLDER: return "0000-AIO" relative_parent = parent.relative_to(IMPORT_FOLDER) return IMPORT_FOLDER.name + "/" + "/".join(relative_parent.parts) def ensure_import_folder_table(cursor, table_name): """Create one folder-specific table using the AIO row layout.""" cursor.execute( f""" CREATE TABLE IF NOT EXISTS {quote_identifier(table_name)} ( ID INTEGER PRIMARY KEY AUTOINCREMENT, DENSECODE TEXT NOT NULL UNIQUE, TAGS TEXT, T2 TEXT, TEXT_TAG TEXT, IMPORT_SOURCE TEXT, RELATIVE_PATH TEXT, T1 TEXT, T3 TEXT, DYNAMIC_DENSECODE TEXT NOT NULL UNIQUE, T4 TEXT ) """ ) def copy_aio_rows_to_import_folder_table(conn, table_name, first_id, last_id): """ Copy only the newly imported AIO rows for one Markdown file into that file's immediate-parent folder table. Densecodes are preserved exactly. """ if not table_name or last_id < first_id: return 0 cursor = conn.cursor() ensure_import_folder_table(cursor, table_name) cursor.execute( f""" SELECT DENSECODE, TAGS, T2, TEXT_TAG, IMPORT_SOURCE, RELATIVE_PATH, T1, T3, DYNAMIC_DENSECODE, T4 FROM {quote_identifier(TABLE_NAME)} WHERE ID BETWEEN ? AND ? ORDER BY ID """, (first_id, last_id) ) rows = cursor.fetchall() if rows: cursor.executemany( f""" INSERT INTO {quote_identifier(table_name)} ( DENSECODE, TAGS, T2, TEXT_TAG, IMPORT_SOURCE, RELATIVE_PATH, T1, T3, DYNAMIC_DENSECODE, T4 ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, rows ) conn.commit() return len(rows) # ============================================================ # IMPORT ALL FILES RECURSIVELY # ============================================================ def import_all_files(conn): md_files = find_markdown_files() log("=" * 70) log("MARKDOWN IMPORT") log("=" * 70) log( f"Markdown files found recursively: " f"{len(md_files)}" ) log("") densecode_counter = 0 dynamic_file_counter = 0 total_rows = 0 total_filenames = 0 total_splits = 0 total_name_blocks = 0 total_equal_filenames = 0 total_tag_lines = 0 total_tags = 0 total_sections = 0 total_dynamic_files = 0 processed_files = 0 folder_tables_used = {} cursor = conn.cursor() for file_path in md_files: processed_files += 1 # Capture the AIO ID range created by this one file so the same # rows can be copied into its immediate-parent folder table. cursor.execute( f'SELECT COALESCE(MAX(ID), 0) FROM {quote_identifier(TABLE_NAME)}' ) before_max_id = cursor.fetchone()[0] proposed_dynamic_file_number = ( dynamic_file_counter + 1 ) ( densecode_counter, imported_rows, filename_count, split_count, name_block_count, equal_filename_count, tag_line_count, tag_count, real_sections ) = import_md_file( conn, file_path, densecode_counter, proposed_dynamic_file_number ) if imported_rows > 0: dynamic_file_counter += 1 total_dynamic_files += 1 # Root files go to 0000-AIO. Nested folders use a root-qualified # / table name. Each level receives only files directly inside it. cursor.execute( f'SELECT COALESCE(MAX(ID), 0) FROM {quote_identifier(TABLE_NAME)}' ) after_max_id = cursor.fetchone()[0] folder_table_name = get_import_folder_table_name(file_path) copied_folder_rows = copy_aio_rows_to_import_folder_table( conn, folder_table_name, before_max_id + 1, after_max_id ) if folder_table_name and copied_folder_rows: folder_tables_used[folder_table_name] = ( folder_tables_used.get(folder_table_name, 0) + copied_folder_rows ) log( f"FOLDER TABLE COPY | TABLE={folder_table_name} | " f"ROWS={copied_folder_rows} | " f"FILE={file_path.relative_to(IMPORT_FOLDER).as_posix()}" ) total_rows += ( imported_rows ) total_filenames += ( filename_count ) total_splits += ( split_count ) total_name_blocks += ( name_block_count ) total_equal_filenames += ( equal_filename_count ) total_tag_lines += ( tag_line_count ) total_tags += ( tag_count ) total_sections += ( real_sections ) log("") log("NORMAL IMPORT FOLDER TABLES") log("-" * 70) if folder_tables_used: for table_name in sorted(folder_tables_used, key=str.casefold): log(f"TABLE={table_name} | ROWS={folder_tables_used[table_name]}") else: log("NONE") log("") return { "files": processed_files, "rows": total_rows, "filenames": total_filenames, "splits": total_splits, "name_blocks": total_name_blocks, "equal_filenames": total_equal_filenames, "tag_lines": total_tag_lines, "tags": total_tags, "sections": total_sections, "dynamic_files": total_dynamic_files, "densecode_counter": densecode_counter, "dynamic_file_counter": dynamic_file_counter } # ============================================================ # VERIFY DATABASE # ============================================================ def verify_database(conn): cursor = conn.cursor() cursor.execute( f''' SELECT COUNT(*) FROM "{TABLE_NAME}" ''' ) row_count = ( cursor.fetchone()[0] ) # ======================================================== # DATABASE ORDER: # # ID # DENSECODE # TAGS # T2 # TEXT_TAG # IMPORT_SOURCE # RELATIVE_PATH # T1 # T3 # DYNAMIC_DENSECODE # T4 # ======================================================== cursor.execute( f''' SELECT ID, DENSECODE, TAGS, T2, TEXT_TAG, IMPORT_SOURCE, RELATIVE_PATH, T1, T3, DYNAMIC_DENSECODE, T4 FROM "{TABLE_NAME}" ORDER BY ID LIMIT 500 ''' ) rows = ( cursor.fetchall() ) log("=" * 70) log("DATABASE VERIFICATION") log("=" * 70) log( f"Rows currently in database: " f"{row_count}" ) log("") for ( row_id, densecode, tags, t2, text_tag, import_source, relative_path, t1, t3, dynamic_densecode, t4 ) in rows: preview = log_t2_preview(t2, 50) log( f"ID={row_id} | " f"DENSECODE={densecode} | " f"T1={t1!r} | " f"T3_PREVIEW_100={log_value_preview(t3, 100)!r} | " f"T4_PREVIEW_100={log_value_preview(t4, 100)!r} | " f"T2_PREVIEW_50={preview!r} | " f"TEXT_TAG_FIRST_10={log_first_text_tags(text_tag)!r} | " f"IMPORT_SOURCE={import_source!r} | " f"RELATIVE_PATH=" f"{relative_path!r} | " f"TAGS={tags!r} | " f"DYNAMIC={dynamic_densecode}" ) log("") return row_count # ============================================================ # CREATE SQL DUMP # ============================================================ def sql_literal(value): """Return a SQLite-safe literal for the SQL export dump.""" if value is None: return "NULL" if isinstance(value, bytes): return "X'" + value.hex() + "'" if isinstance(value, (int, float)): return str(value) text = str(value).replace("'", "''") return "'" + text + "'" def create_sql_dump(conn): log("=" * 70) log("SQL DUMP") log("=" * 70) conn.commit() # Recreate the SQL dump every run. if SQL_DUMP_FILE.exists(): SQL_DUMP_FILE.unlink() cursor = conn.cursor() # Keep the SQL INSERT column order exactly synchronized with the DB. dump_columns = [ "ID", "DENSECODE", "TAGS", "T2", "TEXT_TAG", "IMPORT_SOURCE", "RELATIVE_PATH", "T1", "T3", "DYNAMIC_DENSECODE", "T4", ] quoted_columns = ", ".join( f'"{column}"' for column in dump_columns ) cursor.execute( f'SELECT {quoted_columns} FROM "{TABLE_NAME}" ORDER BY "ID"' ) rows = cursor.fetchall() cursor.execute( "SELECT sql FROM sqlite_master " "WHERE type='table' AND name=?", (TABLE_NAME,) ) schema_row = cursor.fetchone() create_table_sql = schema_row[0] if schema_row else None dump_lines = [ "-- ========================================================", "-- SQLITE DATABASE DUMP", f"-- SOURCE DATABASE: {SQLITE_DB.name}", f"-- CREATED: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", "-- ========================================================", "", "PRAGMA foreign_keys=OFF;", "BEGIN TRANSACTION;", "", ] if create_table_sql: dump_lines.append(create_table_sql.rstrip(";") + ";") dump_lines.append("") for row in rows: values_sql = ", ".join( sql_literal(value) for value in row ) dump_lines.append( f'INSERT INTO "main"."{TABLE_NAME}" ' f'({quoted_columns}) VALUES ({values_sql});' ) # Dump X_TABLE_GOOGLE, populated directly from X files while preserving AIO codes. # DENSECODE and DYNAMIC_DENSECODE remain the original AIO-assigned values. x_columns = [ "ID", "DENSECODE", "TAGS", "T2", "TEXT_TAG", "RELATIVE_PATH", "T1", "T3", "DYNAMIC_DENSECODE", "T4", ] x_quoted_columns = ", ".join( f'"{column}"' for column in x_columns ) cursor.execute( "SELECT sql FROM sqlite_master " "WHERE type='table' AND name=?", (X_TABLE_NAME,) ) x_schema_row = cursor.fetchone() x_create_table_sql = x_schema_row[0] if x_schema_row else None if x_create_table_sql: dump_lines.append("") dump_lines.append(x_create_table_sql.rstrip(";") + ";") dump_lines.append("") cursor.execute( f'SELECT {x_quoted_columns} FROM "{X_TABLE_NAME}" ORDER BY "ID"' ) x_rows = cursor.fetchall() for row in x_rows: values_sql = ", ".join( sql_literal(value) for value in row ) dump_lines.append( f'INSERT INTO "main"."{X_TABLE_NAME}" ' f'({x_quoted_columns}) VALUES ({values_sql});' ) # Dump folder-grouped X tables created from 00-X-IMPORT. # Keep the master AIO table first whenever this script enumerates # database tables. Remaining tables stay alphabetically ordered. cursor.execute( "SELECT name FROM sqlite_master WHERE type='table' " "ORDER BY CASE WHEN name = ? THEN 0 ELSE 1 END, name COLLATE NOCASE", (TABLE_NAME,) ) all_table_names = [row[0] for row in cursor.fetchall()] excluded_group_tables = { TABLE_NAME, X_TABLE_NAME, get_derecho_table_name(), "sqlite_sequence", } x_group_table_names = [] if X_MAIN_TABLE_NAME in all_table_names: x_group_table_names.append(X_MAIN_TABLE_NAME) if X_IMPORT_FOLDER.exists(): for child in sorted(X_IMPORT_FOLDER.iterdir(), key=lambda p: p.name.casefold()): if ( child.is_dir() and child.name in all_table_names and child.name not in excluded_group_tables and child.name not in x_group_table_names ): x_group_table_names.append(child.name) for group_table_name in x_group_table_names: cursor.execute( "SELECT sql FROM sqlite_master WHERE type='table' AND name=?", (group_table_name,) ) group_schema_row = cursor.fetchone() group_create_sql = group_schema_row[0] if group_schema_row else None if group_create_sql: dump_lines.append("") dump_lines.append(group_create_sql.rstrip(";") + ";") dump_lines.append("") cursor.execute( f'SELECT {x_quoted_columns} FROM {quote_identifier(group_table_name)} ORDER BY "ID"' ) group_rows = cursor.fetchall() for row in group_rows: values_sql = ", ".join(sql_literal(value) for value in row) dump_lines.append( f'INSERT INTO "main".{quote_identifier(group_table_name)} ' f'({x_quoted_columns}) VALUES ({values_sql});' ) # Preserve AUTOINCREMENT state when present. try: cursor.execute( "SELECT name, seq FROM sqlite_sequence WHERE name=?", (TABLE_NAME,) ) sequence_row = cursor.fetchone() except sqlite3.OperationalError: sequence_row = None if sequence_row: dump_lines.extend([ "", f'DELETE FROM "sqlite_sequence" WHERE "name"={sql_literal(TABLE_NAME)};', f'INSERT INTO "sqlite_sequence" ("name", "seq") VALUES ' f'({sql_literal(sequence_row[0])}, {sql_literal(sequence_row[1])});', ]) dump_lines.extend([ "", "COMMIT;", "", ]) SQL_DUMP_FILE.write_text( "\n".join(dump_lines), encoding="utf-8" ) dump_size = SQL_DUMP_FILE.stat().st_size insert_count = len(rows) write_export_log( SQL_DUMP_LOG_FILE, "00-EXPORT.sql EXPORT LOG", [ f"SQL file: {SQL_DUMP_FILE.resolve()}", f"Database: {SQLITE_DB.resolve()}", f"AIO inserts: {insert_count}", f"AIO columns included ({len(dump_columns)}): " + ", ".join(dump_columns), f"{X_TABLE_NAME} inserts: {len(x_rows)}", f"{X_TABLE_NAME} columns included ({len(x_columns)}): " + ", ".join(x_columns), f"SQL size: {dump_size} bytes", ], ) shutil.copy2(SQL_DUMP_LOG_FILE, ROOT_SQL_DUMP_LOG) log(f"SQL dump created : {SQL_DUMP_FILE.resolve()}") log(f"Dump size : {dump_size} bytes") log(f"AIO inserts : {insert_count}") log(f"{X_TABLE_NAME} inserts: {len(x_rows)}") log( "AIO INSERT columns: " + ", ".join(dump_columns) ) log("") return { "size": dump_size, "inserts": insert_count } # ============================================================ # CREATE MARKDOWN EXPORT # ============================================================ def create_md_export(conn): log("=" * 70) log("MARKDOWN EXPORT") log("=" * 70) conn.commit() cursor = conn.cursor() cursor.execute( f'SELECT ID, DENSECODE, TAGS, T3 FROM "{TABLE_NAME}" ORDER BY ID' ) rows = cursor.fetchall() # Markdown export format: DENSECODE >>>> TEXT md_lines = [ f"{str(densecode or '')} >>>> {str(text or '')}" for _, densecode, tags, text in rows ] # Recreate the Markdown export every run. if MD_EXPORT_FILE.exists(): MD_EXPORT_FILE.unlink() MD_EXPORT_FILE.write_text( "\n".join(md_lines) + ("\n" if md_lines else ""), encoding="utf-8" ) export_size = MD_EXPORT_FILE.stat().st_size # Recreate a dedicated export containing ONLY the T4 column. if T4_EXPORT_FILE.exists(): T4_EXPORT_FILE.unlink() T4_EXPORT_FILE.write_text( "\n".join(md_lines) + ("\n" if md_lines else ""), encoding="utf-8" ) t4_export_size = T4_EXPORT_FILE.stat().st_size # Recreate the dedicated T4 export log every run. if T4_EXPORT_LOG_FILE.exists(): T4_EXPORT_LOG_FILE.unlink() t4_log_lines = [ "=" * 70, "T4 COLUMN EXPORT LOG", "=" * 70, f"Created: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", f"Database: {SQLITE_DB.resolve()}", f"Markdown export: {MD_EXPORT_FILE.resolve()}", f"T4 column export: {T4_EXPORT_FILE.resolve()}", f"Rows exported: {len(rows)}", "=" * 70, "" ] for row_id, densecode, tags, text in rows: t4_log_lines.append( f"ID={row_id} | DENSECODE={str(densecode or '')} | TAGS={str(tags or '')} | TEXT={str(text or '')}" ) T4_EXPORT_LOG_FILE.write_text( "\n".join(t4_log_lines) + "\n", encoding="utf-8" ) t4_log_size = T4_EXPORT_LOG_FILE.stat().st_size log(f"MD export created : {MD_EXPORT_FILE.resolve()}") log(f"MD rows exported : {len(md_lines)}") log(f"MD export size : {export_size} bytes") log(f"T4 column export : {T4_EXPORT_FILE.resolve()}") log(f"T4 export rows : {len(rows)}") log(f"T4 export size : {t4_export_size} bytes") log(f"T4 export log : {T4_EXPORT_LOG_FILE.resolve()}") log(f"T4 log rows : {len(rows)}") log(f"T4 log size : {t4_log_size} bytes") log("") return { "rows": len(md_lines), "size": export_size, "t4_export_rows": len(rows), "t4_export_size": t4_export_size, "t4_log_rows": len(rows), "t4_log_size": t4_log_size } # ============================================================ # CREATE FULL AIO CSV EXPORT # ============================================================ def create_full_csv_export(conn): """Recreate 00-EXPORT/00-EXPORT.csv from the complete AIO table.""" log("=" * 70) log("FULL AIO CSV EXPORT") log("=" * 70) conn.commit() cursor = conn.cursor() cursor.execute(f'PRAGMA table_info("{TABLE_NAME}")') columns = [row[1] for row in cursor.fetchall()] cursor.execute(f'SELECT * FROM "{TABLE_NAME}" ORDER BY ID') rows = cursor.fetchall() if FULL_EXPORT_CSV.exists(): FULL_EXPORT_CSV.unlink() with FULL_EXPORT_CSV.open("w", newline="", encoding="utf-8-sig") as handle: writer = csv.writer(handle) writer.writerow(columns) writer.writerows(rows) size = FULL_EXPORT_CSV.stat().st_size write_export_log( FULL_EXPORT_CSV_LOG_FILE, "00-EXPORT.csv EXPORT LOG", [ f"CSV file: {FULL_EXPORT_CSV.resolve()}", f"Table: {TABLE_NAME}", f"Rows exported: {len(rows)}", f"Columns exported: {len(columns)}", "Columns included: " + ", ".join(columns), f"CSV size: {size} bytes", ], ) shutil.copy2(FULL_EXPORT_CSV_LOG_FILE, ROOT_EXPORT_CSV_LOG) log(f"Full CSV created : {FULL_EXPORT_CSV.resolve()}") log(f"Full CSV rows : {len(rows)}") log(f"Full CSV columns : {len(columns)}") log(f"Full CSV size : {size} bytes") log("") return {"rows": len(rows), "columns": len(columns), "size": size} # ============================================================ # CREATE 3-COLUMN T4 DB + CSV EXPORT # ============================================================ def create_t4_two_column_exports(conn): log("=" * 70) log("T4 THREE-COLUMN DB + CSV EXPORT") log("=" * 70) conn.commit() cursor = conn.cursor() cursor.execute( f''' SELECT DYNAMIC_DENSECODE, T2, T3, TAGS FROM "{TABLE_NAME}" ORDER BY ID ''' ) rows = cursor.fetchall() # Recreate the dedicated 2-column SQLite DB every run. if T4_TWO_COLUMN_DB.exists(): T4_TWO_COLUMN_DB.unlink() export_conn = sqlite3.connect(T4_TWO_COLUMN_DB) try: export_cursor = export_conn.cursor() export_cursor.execute( 'CREATE TABLE "AIO" (T1 TEXT, T2 TEXT, T3 TEXT)' ) export_cursor.executemany( 'INSERT INTO "AIO" (T1, T2, T3) VALUES (?, ?, ?)', [ ( str(dynamic_code or ""), " ".join(str(source_content or "").split()), str(tags or "") ) for dynamic_code, source_content, limited_flattened_content, tags in rows ] ) export_conn.commit() finally: export_conn.close() # Recreate the dedicated 2-column CSV every run. if T4_TWO_COLUMN_CSV.exists(): T4_TWO_COLUMN_CSV.unlink() with T4_TWO_COLUMN_CSV.open( "w", encoding="utf-8-sig", newline="" ) as csv_handle: writer = csv.writer(csv_handle) writer.writerow(["T1", "T2", "T3"]) writer.writerows( [ ( str(dynamic_code or ""), str(limited_flattened_content or ""), str(tags or "") ) for dynamic_code, source_content, limited_flattened_content, tags in rows ] ) db_size = T4_TWO_COLUMN_DB.stat().st_size csv_size = T4_TWO_COLUMN_CSV.stat().st_size log(f"3-column DB created : {T4_TWO_COLUMN_DB.resolve()}") log(f"3-column CSV created: {T4_TWO_COLUMN_CSV.resolve()}") log(f"Rows exported : {len(rows)}") log("T1 source : DYNAMIC_DENSECODE") log("T2 source (DB) : AIO.T2 flattened during export - FULL CONTENT - NO CHARACTER LIMIT") log(f"T2 source (CSV) : T3 flattened content - {T3_MAX_CHARACTERS} CHARACTER LIMIT") log("T3 source : TAGS") log(f"DB size : {db_size} bytes") log(f"CSV size : {csv_size} bytes") log("") return { "rows": len(rows), "db_size": db_size, "csv_size": csv_size } # ============================================================ # MAIN # ============================================================ def main(): ensure_output_folders() recreate_log_files() LOG_LINES.clear() reset_log_file() reset_x_log_file() # -------------------------------------------------------- # START # -------------------------------------------------------- log("=" * 70) log("MD -> SQLITE IMPORT") log("=" * 70) log( "Started: " + datetime.now().strftime( "%Y-%m-%d %H:%M:%S" ) ) log("") log("=" * 70) log("PARSER FEATURES") log("=" * 70) log("+++ SECTION PARSER ENABLED : YES") log("/// FILENAME PARSER ENABLED: YES") log("= FILENAME PARSER ENABLED : YES") log("TAG PARSER ENABLED : YES") log("MULTI TAG LINES ENABLED : YES") log("SQL DUMP ENABLED : YES") log("MARKDOWN EXPORT ENABLED : YES") log("T4 EXPORT LOG ENABLED : YES") log("T4 2-COLUMN DB EXPORT : YES") log("T4 2-COLUMN CSV EXPORT : YES") log("DYNAMIC DENSECODE ENABLED : YES") log("RECURSIVE *.MD WALK : YES") log("00-IMPORT FOLDER TABLES : YES - 0000-AIO + 00-IMPORT/FOLDER/NESTED") log("LIVE ROOT LOG ENABLED : YES") log("DERECHO ROOT SINGLE TABLE : YES") log("X ROOT -> AIO + X_TABLE_GOOGLE: YES") log("X GROUP TABLE PREFIX ZX- : DISABLED - NO DUPLICATE X TABLES") log("X QUALIFIED FOLDER TABLES : YES - ONE EXACT TABLE PER FOLDER") log("LOG T2 PREVIEW MAX : 50 CHARS") log("LOG T3 PREVIEW MAX : 100 CHARS") log("LOG T4 PREVIEW MAX : 100 CHARS") log("LOG TEXT_TAG MAX : FIRST 10 TAGS") log("X ROOT ACCEPTS *.MD + *.ND : YES") log("X ROOT SEPARATE LOG : YES") log("SINGLE 00-EXPORT FOLDER : YES") log("DEDICATED 00-LOG FOLDER : YES") log("MASTER LOG CONCATENATION : YES") log("LOGS RECREATED EVERY RUN : YES") log("FULL 00-EXPORT.csv : YES") log("ROOT DB + CSV COPIES : YES") log("ROOT MASTER.log COPY : YES") log("00-EXPORT.csv LOG : YES") log("00-EXPORT.sql LOG : YES") log("ROOT CSV + SQL LOG COPIES : YES") log("") # -------------------------------------------------------- # IMPORT ROOT # -------------------------------------------------------- create_import_folder() # -------------------------------------------------------- # CREATE SAMPLE FILES # -------------------------------------------------------- create_sample_files() # -------------------------------------------------------- # DATABASE # -------------------------------------------------------- conn = recreate_database() try: # ---------------------------------------------------- # IMPORT # ---------------------------------------------------- stats = import_all_files( conn ) x_stats = import_x_files( conn, stats["densecode_counter"], stats["dynamic_file_counter"] ) derecho_stats = import_derecho_tables( conn ) # ---------------------------------------------------- # VERIFY # ---------------------------------------------------- database_rows = ( verify_database( conn ) ) # ---------------------------------------------------- # SQL DUMP # ---------------------------------------------------- dump_stats = ( create_sql_dump( conn ) ) # ---------------------------------------------------- # MARKDOWN EXPORT # ---------------------------------------------------- md_export_stats = ( create_md_export( conn ) ) # ---------------------------------------------------- # FULL AIO CSV EXPORT # ---------------------------------------------------- full_csv_stats = ( create_full_csv_export( conn ) ) # ---------------------------------------------------- # T4 2-COLUMN DB + CSV EXPORT # ---------------------------------------------------- two_column_stats = ( create_t4_two_column_exports( conn ) ) finally: conn.close() # Recreate convenient project-root copies after SQLite is fully closed. shutil.copy2(SQLITE_DB, ROOT_SQLITE_DB) shutil.copy2(FULL_EXPORT_CSV, ROOT_EXPORT_CSV) # -------------------------------------------------------- # FINAL RESULTS # -------------------------------------------------------- log("=" * 70) log("RESULT") log("=" * 70) log( f"Markdown files processed : " f"{stats['files']}" ) log( f"MD files with DB rows : " f"{stats['dynamic_files']}" ) log( f"Rows imported : " f"{stats['rows']}" ) log( f"+++ splits detected : " f"{stats['splits']}" ) log( f"Non-empty sections : " f"{stats['sections']}" ) log( f"Filename aliases total : " f"{stats['filenames']}" ) log( f"/// blocks detected : " f"{stats['name_blocks']}" ) log( f"= filenames detected : " f"{stats['equal_filenames']}" ) log( f"Tag lines detected : " f"{stats['tag_lines']}" ) log( f"Tags detected : " f"{stats['tags']}" ) log( f"X files processed : " f"{x_stats['files']}" ) log( f"X rows added to AIO : " f"{x_stats['rows']}" ) log( f"X_TABLE_GOOGLE rows : " f"{x_stats['x_table_rows']}" ) log( f"Database rows : " f"{database_rows}" ) log( f"SQL dump inserts : " f"{dump_stats['inserts']}" ) log( f"Markdown rows exported : " f"{md_export_stats['rows']}" ) log( f"T4 column rows exported : " f"{md_export_stats['t4_export_rows']}" ) log( f"T4 log rows exported : " f"{md_export_stats['t4_log_rows']}" ) log( f"T4 2-column rows exported: " f"{two_column_stats['rows']}" ) log( f"Full CSV rows exported : " f"{full_csv_stats['rows']}" ) log("") log( f"Database : " f"{SQLITE_DB.resolve()}" ) log( f"SQL dump : " f"{SQL_DUMP_FILE.resolve()}" ) log( f"Full CSV : " f"{FULL_EXPORT_CSV.resolve()}" ) log( f"Root DB copy : " f"{ROOT_SQLITE_DB.resolve()}" ) log( f"Root CSV copy : " f"{ROOT_EXPORT_CSV.resolve()}" ) log( f"Markdown export : " f"{MD_EXPORT_FILE.resolve()}" ) log( f"T4 column export : " f"{T4_EXPORT_FILE.resolve()}" ) log( f"T4 export log : " f"{T4_EXPORT_LOG_FILE.resolve()}" ) log( f"T4 2-column database : " f"{T4_TWO_COLUMN_DB.resolve()}" ) log( f"T4 2-column CSV : " f"{T4_TWO_COLUMN_CSV.resolve()}" ) log( f"Import folder : " f"{IMPORT_FOLDER.resolve()}" ) log( f"X import folder : " f"{X_IMPORT_FOLDER.resolve()}" ) log( f"X table log : " f"{X_LOG_FILE.resolve()}" ) log( f"Log file : " f"{LOG_FILE.resolve()}" ) log( f"Export folder : " f"{EXPORT_FOLDER.resolve()}" ) log( f"Log folder : " f"{LOG_FOLDER.resolve()}" ) log( f"Master log : " f"{MASTER_LOG_FILE.resolve()}" ) log( f"CSV export log : " f"{FULL_EXPORT_CSV_LOG_FILE.resolve()}" ) log( f"SQL export log : " f"{SQL_DUMP_LOG_FILE.resolve()}" ) log( f"Root CSV export log copy : " f"{ROOT_EXPORT_CSV_LOG.resolve()}" ) log( f"Root SQL export log copy : " f"{ROOT_SQL_DUMP_LOG.resolve()}" ) log("") log( "Finished: " + datetime.now().strftime( "%Y-%m-%d %H:%M:%S" ) ) log("=" * 70) write_log() master_count = build_master_log() shutil.copy2(MASTER_LOG_FILE, ROOT_MASTER_LOG) print( f"MASTER.log created: {MASTER_LOG_FILE.resolve()} " f"| included {master_count} log files" ) print(f"Root MASTER.log copy: {ROOT_MASTER_LOG.resolve()}") print(f"Root DB copy : {ROOT_SQLITE_DB.resolve()}") print(f"Root CSV copy : {ROOT_EXPORT_CSV.resolve()}") print(f"Root CSV log copy : {ROOT_EXPORT_CSV_LOG.resolve()}") print(f"Root SQL log copy : {ROOT_SQL_DUMP_LOG.resolve()}") # ============================================================ # RUN # ============================================================ if __name__ == "__main__": main()