import csv
import re
import shutil
import sqlite3

from pathlib import Path
from datetime import datetime


# ============================================================
# VARIABLES
# ============================================================

ROOT_FOLDER = Path.cwd()

IMPORT_FOLDER = (
    ROOT_FOLDER
    / "IMPORT-AIO"
)

MD_FOLDER = (
    IMPORT_FOLDER
    / "01-MD"
)

CSV_FOLDER = (
    IMPORT_FOLDER
    / "02-CSV"
)

SQLITE_DB = (
    ROOT_FOLDER
    / "00-IMPORT-AIO.db"
)


# ============================================================
# EXPORT DUMP
# ============================================================

EXPORT_DUMP_FOLDER = (
    ROOT_FOLDER
    / "00-EXPORT-DUMP"
)

EXPORT_DUMP_EXTENSION = ".txt"

EXPORT_FILENAME_SEPARATOR = " --- "


# ============================================================
# WINDOWS FILE NAME LIMIT
# ============================================================

MAX_EXPORT_FILENAME_LENGTH = 220

MAX_NAME_MARK_FILENAME_LENGTH = 100


# ============================================================
# SAMPLE VARIABLES
# ============================================================

MIN_UNUSED_MD_SAMPLES = 10

MIN_UNUSED_CSV_SAMPLES = 10

MD_SAMPLE_PREFIX = "SAMPLE"

CSV_SAMPLE_PREFIX = "SAMPLE"


# ============================================================
# CSV VARIABLES
# ============================================================

CSV_TEXT_COLUMNS = [
    f"T{i}"
    for i in range(
        1,
        51
    )
]


CSV_SAMPLE_COLUMNS = [
    "NAME_MARK",
    "DESCRIPTION",
    "X",
    *CSV_TEXT_COLUMNS,
]


CSV_CONCAT_DELIMITER = " "


# ============================================================
# X VARIABLES
# ============================================================

X_MARKER = "///"

X_GROUP_LINE_DELIMITER = "\n"


# ============================================================
# *** VARIABLES
# ============================================================

TRIPLE_ASTERISK_MARKER = "***"

TRIPLE_ASTERISK_REPLACEMENT = "\n\n"


# ============================================================
# LOG FILES
# ============================================================

GENERAL_LOG_FILE = (
    ROOT_FOLDER
    / "00-IMPORT-FOLDER-SAMPLE-CREATION.log"
)

MD_LOG_FILE = (
    ROOT_FOLDER
    / "00-MD-PROCESSING.log"
)

CSV_LOG_FILE = (
    ROOT_FOLDER
    / "00-CSV-PROCESSING.log"
)

X_LOG_FILE = (
    ROOT_FOLDER
    / "00-X-PROCESSING.log"
)

X_TABLE_LOG_FILE = (
    ROOT_FOLDER
    / "00-X-TABLE-PROCESSING.log"
)

TAGS_LOG_FILE = (
    ROOT_FOLDER
    / "00-TAGS-PROCESSING.log"
)

NAME_MARK_LOG_FILE = (
    ROOT_FOLDER
    / "00-NAME-MARK-PROCESSING.log"
)

DESCRIPTION_LOG_FILE = (
    ROOT_FOLDER
    / "00-DESCRIPTION-PROCESSING.log"
)

TRIPLE_ASTERISK_LOG_FILE = (
    ROOT_FOLDER
    / "00-TRIPLE-ASTERISK-PROCESSING.log"
)

DENSE_CODE_LOG_FILE = (
    ROOT_FOLDER
    / "00-DENSE-CODE.log"
)

EXPORT_DUMP_LOG_FILE = (
    ROOT_FOLDER
    / "00-EXPORT-DUMP-PROCESSING.log"
)


# ============================================================
# LOG STORAGE
# ============================================================

GENERAL_LOG_LINES = []

MD_LOG_LINES = []

CSV_LOG_LINES = []

X_LOG_LINES = []

X_TABLE_LOG_LINES = []

TAGS_LOG_LINES = []

NAME_MARK_LOG_LINES = []

DESCRIPTION_LOG_LINES = []

TRIPLE_ASTERISK_LOG_LINES = []

DENSE_CODE_LOG_LINES = []

EXPORT_DUMP_LOG_LINES = []


# ============================================================
# LOG FILE MAP
# ============================================================

LOG_FILE_MAP = {}


# ============================================================
# INITIALIZE LOG FILE MAP
# ============================================================

def initialize_log_file_map():

    global LOG_FILE_MAP


    LOG_FILE_MAP = {

        id(
            GENERAL_LOG_LINES
        ):
        GENERAL_LOG_FILE,

        id(
            MD_LOG_LINES
        ):
        MD_LOG_FILE,

        id(
            CSV_LOG_LINES
        ):
        CSV_LOG_FILE,

        id(
            X_LOG_LINES
        ):
        X_LOG_FILE,

        id(
            X_TABLE_LOG_LINES
        ):
        X_TABLE_LOG_FILE,

        id(
            TAGS_LOG_LINES
        ):
        TAGS_LOG_FILE,

        id(
            NAME_MARK_LOG_LINES
        ):
        NAME_MARK_LOG_FILE,

        id(
            DESCRIPTION_LOG_LINES
        ):
        DESCRIPTION_LOG_FILE,

        id(
            TRIPLE_ASTERISK_LOG_LINES
        ):
        TRIPLE_ASTERISK_LOG_FILE,

        id(
            DENSE_CODE_LOG_LINES
        ):
        DENSE_CODE_LOG_FILE,

        id(
            EXPORT_DUMP_LOG_LINES
        ):
        EXPORT_DUMP_LOG_FILE,
    }


# ============================================================
# LOG HELPERS
# ============================================================

def make_log_line(
    message
):

    timestamp = (
        datetime.now()
        .strftime(
            "%Y-%m-%d %H:%M:%S"
        )
    )


    return (
        f"[{timestamp}] "
        f"{message}"
    )


def append_log(
    storage,
    message
):

    line = make_log_line(
        message
    )


    storage.append(
        line
    )


    print(
        line
    )


    log_file = LOG_FILE_MAP.get(
        id(
            storage
        )
    )


    if log_file is None:

        return


    try:

        with open(
            log_file,
            "a",
            encoding="utf-8"
        ) as file:

            file.write(
                line
                +
                "\n"
            )


            file.flush()


    except Exception as error:

        print(
            "[LOG WRITE ERROR]"
            " | "
            f"FILE={log_file}"
            " | "
            f"ERROR={error}"
        )


def log(
    message
):

    append_log(
        GENERAL_LOG_LINES,
        message
    )


def md_log(
    message
):

    append_log(
        MD_LOG_LINES,
        message
    )


def csv_log(
    message
):

    append_log(
        CSV_LOG_LINES,
        message
    )


def x_log(
    message
):

    append_log(
        X_LOG_LINES,
        message
    )


def x_table_log(
    message
):

    append_log(
        X_TABLE_LOG_LINES,
        message
    )


def tags_log(
    message
):

    append_log(
        TAGS_LOG_LINES,
        message
    )


def name_mark_log(
    message
):

    append_log(
        NAME_MARK_LOG_LINES,
        message
    )


def description_log(
    message
):

    append_log(
        DESCRIPTION_LOG_LINES,
        message
    )


def triple_log(
    message
):

    append_log(
        TRIPLE_ASTERISK_LOG_LINES,
        message
    )


def dense_log(
    message
):

    append_log(
        DENSE_CODE_LOG_LINES,
        message
    )


def export_dump_log(
    message
):

    append_log(
        EXPORT_DUMP_LOG_LINES,
        message
    )


# ============================================================
# RECREATE LOG FILES
# ============================================================

def recreate_log_files():

    log_files = [
        GENERAL_LOG_FILE,
        MD_LOG_FILE,
        CSV_LOG_FILE,
        X_LOG_FILE,
        X_TABLE_LOG_FILE,
        TAGS_LOG_FILE,
        NAME_MARK_LOG_FILE,
        DESCRIPTION_LOG_FILE,
        TRIPLE_ASTERISK_LOG_FILE,
        DENSE_CODE_LOG_FILE,
        EXPORT_DUMP_LOG_FILE,
    ]


    GENERAL_LOG_LINES.clear()

    MD_LOG_LINES.clear()

    CSV_LOG_LINES.clear()

    X_LOG_LINES.clear()

    X_TABLE_LOG_LINES.clear()

    TAGS_LOG_LINES.clear()

    NAME_MARK_LOG_LINES.clear()

    DESCRIPTION_LOG_LINES.clear()

    TRIPLE_ASTERISK_LOG_LINES.clear()

    DENSE_CODE_LOG_LINES.clear()

    EXPORT_DUMP_LOG_LINES.clear()


    for file_path in log_files:

        try:

            if file_path.exists():

                file_path.unlink()


            file_path.parent.mkdir(
                parents=True,
                exist_ok=True
            )


            file_path.touch()


        except Exception as error:

            print(
                "[LOG RECREATE ERROR]"
                " | "
                f"FILE={file_path}"
                " | "
                f"ERROR={error}"
            )


    initialize_log_file_map()


    log(
        "GENERAL LOG FILE RECREATED: "
        f"{GENERAL_LOG_FILE.name}"
    )


    md_log(
        "MD LOG FILE RECREATED: "
        f"{MD_LOG_FILE.name}"
    )


    csv_log(
        "CSV LOG FILE RECREATED: "
        f"{CSV_LOG_FILE.name}"
    )


    x_log(
        "X LOG FILE RECREATED: "
        f"{X_LOG_FILE.name}"
    )


    x_table_log(
        "X_TABLE LOG FILE RECREATED: "
        f"{X_TABLE_LOG_FILE.name}"
    )


    tags_log(
        "TAGS LOG FILE RECREATED: "
        f"{TAGS_LOG_FILE.name}"
    )


    name_mark_log(
        "NAME_MARK LOG FILE RECREATED: "
        f"{NAME_MARK_LOG_FILE.name}"
    )


    description_log(
        "DESCRIPTION LOG FILE RECREATED: "
        f"{DESCRIPTION_LOG_FILE.name}"
    )


    triple_log(
        "*** LOG FILE RECREATED: "
        f"{TRIPLE_ASTERISK_LOG_FILE.name}"
    )


    dense_log(
        "DENSE CODE LOG FILE RECREATED: "
        f"{DENSE_CODE_LOG_FILE.name}"
    )


    export_dump_log(
        "EXPORT DUMP LOG FILE RECREATED: "
        f"{EXPORT_DUMP_LOG_FILE.name}"
    )


# ============================================================
# SAVE LOGS
#
# FILES ARE ALREADY UPDATED IMMEDIATELY
# ============================================================

def save_logs():

    log(
        "ALL LOG FILES WERE WRITTEN IMMEDIATELY"
    )


# ============================================================
# CREATE IMPORT FOLDERS
# ============================================================

def create_import_folders():

    folders = [
        IMPORT_FOLDER,
        MD_FOLDER,
        CSV_FOLDER,
    ]


    for folder in folders:

        existed = folder.exists()


        folder.mkdir(
            parents=True,
            exist_ok=True
        )


        relative_path = (
            folder
            .relative_to(
                ROOT_FOLDER
            )
            .as_posix()
        )


        if existed:

            log(
                "FOLDER EXISTS"
                " | "
                f"{relative_path}"
            )

        else:

            log(
                "FOLDER CREATED"
                " | "
                f"{relative_path}"
            )


# ============================================================
# DATABASE
#
# REBUILD ONLY IMPORT_AIO
#
# PRESERVE OTHER UNRELATED TABLES
#
# NO AUTOINCREMENT
# ============================================================

def create_sqlite_database():

    connection = sqlite3.connect(
        SQLITE_DB
    )

    cursor = connection.cursor()


    cursor.execute(
        """
        DROP TABLE IF EXISTS MD_IMPORT
        """
    )


    cursor.execute(
        """
        DROP TABLE IF EXISTS CSV_IMPORT
        """
    )


    cursor.execute(
        """
        DROP TABLE IF EXISTS SAMPLE_FILES
        """
    )


    cursor.execute(
        """
        DROP TABLE IF EXISTS IMPORT_AIO
        """
    )


    cursor.execute(
        """
        CREATE TABLE IMPORT_AIO
        (
            ID INTEGER PRIMARY KEY,

            SOURCE_TYPE TEXT,

            SOURCE_FILE TEXT,

            SOURCE_ROW INTEGER,

            SECTION_NUMBER INTEGER,

            TEXT_CONTENT TEXT,

            TAGS TEXT,

            NAME_MARK TEXT,

            DESCRIPTION TEXT,

            X TEXT,

            DENSE_CODE TEXT
        )
        """
    )


    connection.commit()

    connection.close()


    log(
        "DATABASE READY"
        " | "
        f"{SQLITE_DB.name}"
    )


    log(
        "IMPORT_AIO TABLE RECREATED"
    )


# ============================================================
# GET ROOT + NESTED FOLDERS
# ============================================================

def get_all_nested_folders(
    base_folder
):

    folders = [
        base_folder
    ]


    for path in base_folder.rglob(
        "*"
    ):

        if path.is_dir():

            folders.append(
                path
            )


    return sorted(
        folders,
        key=lambda path: (
            len(
                path.parts
            ),
            path.as_posix().lower()
        )
    )


# ============================================================
# UNUSED MD SAMPLE
# ============================================================

def md_is_unused(
    file_path
):

    try:

        content = file_path.read_text(
            encoding="utf-8",
            errors="ignore"
        )


        return (
            content.strip()
            ==
            ""
        )


    except Exception:

        return False


# ============================================================
# UNUSED CSV SAMPLE
# ============================================================

def csv_is_unused(
    file_path
):

    try:

        with open(
            file_path,
            "r",
            encoding="utf-8-sig",
            newline=""
        ) as file:

            reader = csv.DictReader(
                file
            )


            if reader.fieldnames is None:

                return True


            for row in reader:

                for value in row.values():

                    if value is None:

                        continue


                    if str(
                        value
                    ).strip():

                        return False


        return True


    except Exception:

        return False


# ============================================================
# GET SAMPLE FILES
#
# CURRENT FOLDER ONLY
# ============================================================

def get_samples_in_folder(
    folder,
    extension,
    prefix
):

    files = []


    for file_path in folder.iterdir():

        if not file_path.is_file():

            continue


        if (
            file_path.suffix.lower()
            !=
            extension.lower()
        ):

            continue


        if not (
            file_path
            .stem
            .upper()
            .startswith(
                prefix.upper()
            )
        ):

            continue


        files.append(
            file_path
        )


    return sorted(
        files
    )


# ============================================================
# NEXT SAMPLE PATH
# ============================================================

def next_sample_path(
    folder,
    prefix,
    extension
):

    number = 1


    while True:

        file_path = (
            folder
            /
            (
                f"{prefix}-"
                f"{number:03d}"
                f"{extension}"
            )
        )


        if not file_path.exists():

            return file_path


        number += 1


# ============================================================
# CREATE MD SAMPLE
# ============================================================

def create_md_sample(
    folder
):

    file_path = next_sample_path(
        folder,
        MD_SAMPLE_PREFIX,
        ".md"
    )


    file_path.write_text(
        "",
        encoding="utf-8"
    )


    log(
        "MD SAMPLE CREATED"
        " | "
        f"FILE="
        f"{file_path.relative_to(ROOT_FOLDER).as_posix()}"
    )


# ============================================================
# CREATE CSV SAMPLE
# ============================================================

def create_csv_sample(
    folder
):

    file_path = next_sample_path(
        folder,
        CSV_SAMPLE_PREFIX,
        ".csv"
    )


    with open(
        file_path,
        "w",
        encoding="utf-8-sig",
        newline=""
    ) as file:

        writer = csv.writer(
            file
        )


        writer.writerow(
            CSV_SAMPLE_COLUMNS
        )


    log(
        "CSV SAMPLE CREATED"
        " | "
        f"FILE="
        f"{file_path.relative_to(ROOT_FOLDER).as_posix()}"
    )


# ============================================================
# ENSURE MD SAMPLES
# ============================================================

def ensure_unused_md_samples():

    folders = get_all_nested_folders(
        MD_FOLDER
    )


    for folder in folders:

        samples = get_samples_in_folder(
            folder,
            ".md",
            MD_SAMPLE_PREFIX
        )


        unused = [
            file_path

            for file_path in samples

            if md_is_unused(
                file_path
            )
        ]


        used = [
            file_path

            for file_path in samples

            if not md_is_unused(
                file_path
            )
        ]


        needed = max(
            0,
            MIN_UNUSED_MD_SAMPLES
            -
            len(
                unused
            )
        )


        relative_folder = (
            folder
            .relative_to(
                ROOT_FOLDER
            )
            .as_posix()
        )


        log(
            "MD FOLDER"
            " | "
            f"FOLDER={relative_folder}"
            " | "
            f"SAMPLES={len(samples)}"
            " | "
            f"USED={len(used)}"
            " | "
            f"UNUSED={len(unused)}"
            " | "
            f"NEEDED={needed}"
        )


        for used_file in used:

            log(
                "USED MD SAMPLE PRESERVED"
                " | "
                f"FILE={used_file.name}"
            )


        for _ in range(
            needed
        ):

            create_md_sample(
                folder
            )


# ============================================================
# ENSURE CSV SAMPLES
# ============================================================

def ensure_unused_csv_samples():

    folders = get_all_nested_folders(
        CSV_FOLDER
    )


    for folder in folders:

        samples = get_samples_in_folder(
            folder,
            ".csv",
            CSV_SAMPLE_PREFIX
        )


        unused = [
            file_path

            for file_path in samples

            if csv_is_unused(
                file_path
            )
        ]


        used = [
            file_path

            for file_path in samples

            if not csv_is_unused(
                file_path
            )
        ]


        needed = max(
            0,
            MIN_UNUSED_CSV_SAMPLES
            -
            len(
                unused
            )
        )


        relative_folder = (
            folder
            .relative_to(
                ROOT_FOLDER
            )
            .as_posix()
        )


        log(
            "CSV FOLDER"
            " | "
            f"FOLDER={relative_folder}"
            " | "
            f"SAMPLES={len(samples)}"
            " | "
            f"USED={len(used)}"
            " | "
            f"UNUSED={len(unused)}"
            " | "
            f"NEEDED={needed}"
        )


        for used_file in used:

            log(
                "USED CSV SAMPLE PRESERVED"
                " | "
                f"FILE={used_file.name}"
            )


        for _ in range(
            needed
        ):

            create_csv_sample(
                folder
            )


# ============================================================
# PROCESS ***
#
# *** -> TWO NEWLINES
# ============================================================

def process_triple_asterisk(
    text,
    source_type,
    source_file,
    source_row=None
):

    if text is None:

        return ""


    text = str(
        text
    )


    amount = text.count(
        TRIPLE_ASTERISK_MARKER
    )


    if amount <= 0:

        return text


    processed_text = text.replace(
        TRIPLE_ASTERISK_MARKER,
        TRIPLE_ASTERISK_REPLACEMENT
    )


    triple_log(
        "*** REPLACED"
        " | "
        f"TYPE={source_type}"
        " | "
        f"FILE={source_file}"
        " | "
        f"ROW={source_row}"
        " | "
        f"COUNT={amount}"
    )


    return processed_text


# ============================================================
# MD +++ SPLIT
# ============================================================

def split_md_sections(
    content
):

    sections = re.split(
        r"^\s*\+\+\+\s*$",
        content,
        flags=re.MULTILINE
    )


    return [
        section

        for section in sections

        if section.strip()
    ]


# ============================================================
# MD TAG EXTRACTION
#
# ///
# APPLE
# BOTTOM
# JEANS
# ///
#
# TAG BLOCK REMOVED FROM TEXT_CONTENT
# ============================================================

def extract_tags_from_section(
    section
):

    tags = []


    pattern = re.compile(
        r"^\s*///\s*$"
        r"(.*?)"
        r"^\s*///\s*$",
        flags=(
            re.MULTILINE
            |
            re.DOTALL
        )
    )


    def remove_tag_block(
        match
    ):

        block = match.group(
            1
        )


        for line in block.splitlines():

            tag = line.strip()


            if tag:

                tags.append(
                    tag
                )


        return "\n"


    cleaned_text = pattern.sub(
        remove_tag_block,
        section
    )


    cleaned_text = re.sub(
        r"\n{3,}",
        "\n\n",
        cleaned_text
    )


    return (
        cleaned_text.strip(),
        tags
    )


# ============================================================
# PROCESS MD FILES
# ============================================================

def process_md_files():

    md_log(
        "========================================"
    )

    md_log(
        "STARTING MD PROCESSING"
    )

    md_log(
        "========================================"
    )


    files = [
        file_path

        for file_path in MD_FOLDER.rglob(
            "*"
        )

        if (
            file_path.is_file()
            and
            file_path.suffix.lower()
            ==
            ".md"
        )
    ]


    connection = sqlite3.connect(
        SQLITE_DB
    )

    cursor = connection.cursor()


    total_rows = 0

    total_tags = 0

    sections_with_tags = 0


    for file_path in sorted(
        files
    ):

        relative_path = (
            file_path
            .relative_to(
                ROOT_FOLDER
            )
            .as_posix()
        )


        md_log(
            "PROCESSING MD"
            " | "
            f"FILE={relative_path}"
        )


        try:

            content = file_path.read_text(
                encoding="utf-8",
                errors="ignore"
            )


        except Exception as error:

            md_log(
                "MD READ ERROR"
                " | "
                f"FILE={relative_path}"
                " | "
                f"ERROR={error}"
            )

            continue


        if not content.strip():

            md_log(
                "SKIPPED EMPTY MD"
                " | "
                f"FILE={relative_path}"
            )

            continue


        content = process_triple_asterisk(
            content,
            "MD",
            relative_path
        )


        sections = split_md_sections(
            content
        )


        md_log(
            "MD SECTIONS"
            " | "
            f"FILE={relative_path}"
            " | "
            f"COUNT={len(sections)}"
        )


        for (
            section_number,
            section
        ) in enumerate(
            sections,
            start=1
        ):

            (
                cleaned_text,
                tags
            ) = extract_tags_from_section(
                section
            )


            tags_text = "\n".join(
                tags
            )


            x_text = tags_text


            if tags:

                sections_with_tags += 1


                tags_log(
                    "MD TAG BLOCK"
                    " | "
                    f"FILE={relative_path}"
                    " | "
                    f"SECTION={section_number}"
                    " | "
                    f"COUNT={len(tags)}"
                )


                for (
                    tag_number,
                    tag
                ) in enumerate(
                    tags,
                    start=1
                ):

                    tags_log(
                        "TAG"
                        " | "
                        f"FILE={relative_path}"
                        " | "
                        f"SECTION={section_number}"
                        " | "
                        f"TAG_NUMBER={tag_number}"
                        " | "
                        f"VALUE={tag}"
                    )


            cursor.execute(
                """
                INSERT INTO IMPORT_AIO
                (
                    SOURCE_TYPE,
                    SOURCE_FILE,
                    SOURCE_ROW,
                    SECTION_NUMBER,
                    TEXT_CONTENT,
                    TAGS,
                    NAME_MARK,
                    DESCRIPTION,
                    X,
                    DENSE_CODE
                )
                VALUES
                (
                    ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
                )
                """,
                (
                    "MD",
                    relative_path,
                    None,
                    section_number,
                    cleaned_text,
                    tags_text,
                    "",
                    "",
                    x_text,
                    None
                )
            )


            total_rows += 1

            total_tags += len(
                tags
            )


            md_log(
                "MD SECTION INSERTED"
                " | "
                f"FILE={relative_path}"
                " | "
                f"SECTION={section_number}"
                " | "
                f"TAG_COUNT={len(tags)}"
            )


    connection.commit()

    connection.close()


    md_log(
        "TOTAL MD FILES FOUND: "
        f"{len(files)}"
    )


    md_log(
        "TOTAL MD ROWS INSERTED: "
        f"{total_rows}"
    )


    tags_log(
        "TOTAL SECTIONS WITH TAGS: "
        f"{sections_with_tags}"
    )


    tags_log(
        "TOTAL TAGS EXTRACTED: "
        f"{total_tags}"
    )


    tags_log(
        "TAGS COPIED TO X COLUMN: "
        f"{total_tags}"
    )


# ============================================================
# NORMALIZE CSV ROW
# ============================================================

def normalize_csv_row(
    row
):

    normalized = {}


    for key, value in row.items():

        if key is None:

            continue


        normalized_key = (
            str(
                key
            )
            .strip()
            .upper()
        )


        normalized[
            normalized_key
        ] = value


    return normalized


# ============================================================
# CONCATENATE T1-T50
#
# NAME_MARK IS NEVER INCLUDED
# ============================================================

def concatenate_csv_text(
    row,
    source_file,
    source_row
):

    parts = []


    for column_name in CSV_TEXT_COLUMNS:

        value = row.get(
            column_name,
            ""
        )


        if value is None:

            continue


        value = str(
            value
        ).strip()


        if value:

            parts.append(
                value
            )


    text = CSV_CONCAT_DELIMITER.join(
        parts
    ).strip()


    if not text:

        return ""


    return process_triple_asterisk(
        text,
        "CSV",
        source_file,
        source_row
    )


# ============================================================
# INSERT CSV_X GROUP
#
# TEXT_CONTENT:
#
# ONLY CONCATENATED T1-T50
#
# NAME_MARK:
#
# CONCATENATED SEPARATELY
#
# X:
#
# NAME_MARK + TEXT
# ============================================================

def insert_x_group(
    cursor,
    relative_path,
    start_row,
    end_row,
    text_lines,
    name_marks,
    descriptions
):

    text_content = (
        X_GROUP_LINE_DELIMITER
        .join(
            text_lines
        )
        .strip()
    )


    if not text_content:

        x_log(
            "EMPTY X GROUP SKIPPED"
            " | "
            f"FILE={relative_path}"
            " | "
            f"START_ROW={start_row}"
            " | "
            f"END_ROW={end_row}"
        )


        return False


    name_mark_text = (
        "\n"
        .join(
            name_marks
        )
        .strip()
    )


    description_text = (
        "\n"
        .join(
            descriptions
        )
        .strip()
    )


    x_parts = []


    if name_mark_text:

        x_parts.append(
            name_mark_text
        )


    if text_content:

        x_parts.append(
            text_content
        )


    x_text = (
        "\n"
        .join(
            x_parts
        )
        .strip()
    )


    cursor.execute(
        """
        INSERT INTO IMPORT_AIO
        (
            SOURCE_TYPE,
            SOURCE_FILE,
            SOURCE_ROW,
            SECTION_NUMBER,
            TEXT_CONTENT,
            TAGS,
            NAME_MARK,
            DESCRIPTION,
            X,
            DENSE_CODE
        )
        VALUES
        (
            ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
        )
        """,
        (
            "CSV_X",
            relative_path,
            start_row,
            None,
            text_content,
            "",
            name_mark_text,
            description_text,
            x_text,
            None
        )
    )


    x_log(
        "CSV_X INSERTED"
        " | "
        f"FILE={relative_path}"
        " | "
        f"START_ROW={start_row}"
        " | "
        f"END_ROW={end_row}"
        " | "
        f"TEXT_LINES={len(text_lines)}"
        " | "
        f"NAME_MARKS={len(name_marks)}"
    )


    return True


# ============================================================
# PROCESS CSV FILES
# ============================================================

def process_csv_files():

    csv_log(
        "========================================"
    )

    csv_log(
        "STARTING CSV PROCESSING"
    )

    csv_log(
        "========================================"
    )


    files = [
        file_path

        for file_path in CSV_FOLDER.rglob(
            "*"
        )

        if (
            file_path.is_file()
            and
            file_path.suffix.lower()
            ==
            ".csv"
        )
    ]


    connection = sqlite3.connect(
        SQLITE_DB
    )

    cursor = connection.cursor()


    total_regular_rows = 0

    total_x_marker_rows = 0

    total_x_groups = 0

    total_x_content_lines = 0

    total_empty_x_groups = 0

    total_name_marks = 0

    total_descriptions = 0


    for file_path in sorted(
        files
    ):

        relative_path = (
            file_path
            .relative_to(
                ROOT_FOLDER
            )
            .as_posix()
        )


        csv_log(
            "PROCESSING CSV"
            " | "
            f"FILE={relative_path}"
        )


        try:

            with open(
                file_path,
                "r",
                encoding="utf-8-sig",
                newline=""
            ) as file:

                reader = csv.DictReader(
                    file
                )


                if reader.fieldnames is None:

                    csv_log(
                        "CSV SKIPPED - NO HEADER"
                        " | "
                        f"FILE={relative_path}"
                    )

                    continue


                x_group_active = False

                x_group_start_row = None

                x_group_end_row = None

                x_group_text_lines = []

                x_group_name_marks = []

                x_group_descriptions = []


                for (
                    source_row_number,
                    raw_row
                ) in enumerate(
                    reader,
                    start=1
                ):

                    row = normalize_csv_row(
                        raw_row
                    )


                    name_mark = str(
                        row.get(
                            "NAME_MARK",
                            ""
                        )
                        or
                        ""
                    ).strip()


                    description = str(
                        row.get(
                            "DESCRIPTION",
                            ""
                        )
                        or
                        ""
                    ).strip()


                    x_value = str(
                        row.get(
                            "X",
                            ""
                        )
                        or
                        ""
                    ).strip()


                    concatenated_text = (
                        concatenate_csv_text(
                            row,
                            relative_path,
                            source_row_number
                        )
                    )


                    # =========================================
                    # NAME_MARK
                    # =========================================

                    if name_mark:

                        total_name_marks += 1


                        name_mark_log(
                            "NAME_MARK FOUND"
                            " | "
                            f"FILE={relative_path}"
                            " | "
                            f"ROW={source_row_number}"
                            " | "
                            f"VALUE={name_mark}"
                        )


                    # =========================================
                    # DESCRIPTION
                    # =========================================

                    if description:

                        total_descriptions += 1


                        description_log(
                            "DESCRIPTION FOUND"
                            " | "
                            f"FILE={relative_path}"
                            " | "
                            f"ROW={source_row_number}"
                            " | "
                            f"VALUE={description}"
                        )


                    # =========================================
                    # X = ///
                    # =========================================

                    if x_value == X_MARKER:

                        total_x_marker_rows += 1


                        if not x_group_active:

                            x_group_active = True

                            x_group_start_row = (
                                source_row_number
                            )

                            x_group_text_lines = []

                            x_group_name_marks = []

                            x_group_descriptions = []


                            x_log(
                                "X GROUP START"
                                " | "
                                f"FILE={relative_path}"
                                " | "
                                f"ROW={source_row_number}"
                            )


                        x_group_end_row = (
                            source_row_number
                        )


                        # -------------------------------------
                        # NAME_MARK SAVED SEPARATELY
                        # -------------------------------------

                        if name_mark:

                            x_group_name_marks.append(
                                name_mark
                            )


                            name_mark_log(
                                "NAME_MARK ADDED TO X GROUP"
                                " | "
                                f"FILE={relative_path}"
                                " | "
                                f"ROW={source_row_number}"
                                " | "
                                f"VALUE={name_mark}"
                            )


                        # -------------------------------------
                        # DESCRIPTION SAVED SEPARATELY
                        # -------------------------------------

                        if description:

                            x_group_descriptions.append(
                                description
                            )


                        # -------------------------------------
                        # TEXT CONTENT
                        # T1-T50 ONLY
                        # -------------------------------------

                        if concatenated_text:

                            x_group_text_lines.append(
                                concatenated_text
                            )


                            total_x_content_lines += 1


                            x_log(
                                "X CONTENT"
                                " | "
                                f"FILE={relative_path}"
                                " | "
                                f"ROW={source_row_number}"
                                " | "
                                f"TEXT={concatenated_text}"
                            )


                        else:

                            x_log(
                                "X EMPTY MARKER ROW"
                                " | "
                                f"FILE={relative_path}"
                                " | "
                                f"ROW={source_row_number}"
                                " | "
                                "GROUP CONTINUES"
                            )


                    # =========================================
                    # BLANK X BREAKS GROUP
                    # =========================================

                    else:

                        if x_group_active:

                            inserted = insert_x_group(
                                cursor,
                                relative_path,
                                x_group_start_row,
                                x_group_end_row,
                                x_group_text_lines,
                                x_group_name_marks,
                                x_group_descriptions
                            )


                            if inserted:

                                total_x_groups += 1


                            else:

                                total_empty_x_groups += 1


                            x_group_active = False

                            x_group_start_row = None

                            x_group_end_row = None

                            x_group_text_lines = []

                            x_group_name_marks = []

                            x_group_descriptions = []


                    # =========================================
                    # REGULAR CSV ROW
                    # =========================================

                    if (
                        concatenated_text
                        or
                        name_mark
                        or
                        description
                    ):

                        # -------------------------------------
                        # NORMAL ROW X:
                        #
                        # NAME_MARK
                        #
                        # TEXT_CONTENT:
                        #
                        # T1-T50 ONLY
                        # -------------------------------------

                        regular_x = (
                            name_mark
                            if name_mark
                            else
                            ""
                        )


                        cursor.execute(
                            """
                            INSERT INTO IMPORT_AIO
                            (
                                SOURCE_TYPE,
                                SOURCE_FILE,
                                SOURCE_ROW,
                                SECTION_NUMBER,
                                TEXT_CONTENT,
                                TAGS,
                                NAME_MARK,
                                DESCRIPTION,
                                X,
                                DENSE_CODE
                            )
                            VALUES
                            (
                                ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
                            )
                            """,
                            (
                                "CSV",
                                relative_path,
                                source_row_number,
                                None,
                                concatenated_text,
                                "",
                                name_mark,
                                description,
                                regular_x,
                                None
                            )
                        )


                        total_regular_rows += 1


                        csv_log(
                            "CSV ROW INSERTED"
                            " | "
                            f"FILE={relative_path}"
                            " | "
                            f"ROW={source_row_number}"
                            " | "
                            f"NAME_MARK={name_mark}"
                        )


                # =============================================
                # EOF FLUSH
                # =============================================

                if x_group_active:

                    inserted = insert_x_group(
                        cursor,
                        relative_path,
                        x_group_start_row,
                        x_group_end_row,
                        x_group_text_lines,
                        x_group_name_marks,
                        x_group_descriptions
                    )


                    if inserted:

                        total_x_groups += 1


                    else:

                        total_empty_x_groups += 1


        except Exception as error:

            csv_log(
                "CSV PROCESSING ERROR"
                " | "
                f"FILE={relative_path}"
                " | "
                f"ERROR={type(error).__name__}: {error}"
            )


    connection.commit()

    connection.close()


    csv_log(
        "TOTAL CSV FILES FOUND: "
        f"{len(files)}"
    )


    csv_log(
        "TOTAL REGULAR CSV ROWS: "
        f"{total_regular_rows}"
    )


    x_log(
        "TOTAL /// ROWS: "
        f"{total_x_marker_rows}"
    )


    x_log(
        "TOTAL X GROUPS INSERTED: "
        f"{total_x_groups}"
    )


    x_log(
        "TOTAL X CONTENT LINES: "
        f"{total_x_content_lines}"
    )


    x_log(
        "EMPTY X GROUPS SKIPPED: "
        f"{total_empty_x_groups}"
    )


    name_mark_log(
        "TOTAL NAME_MARK VALUES FOUND: "
        f"{total_name_marks}"
    )


    description_log(
        "TOTAL DESCRIPTION VALUES FOUND: "
        f"{total_descriptions}"
    )


# ============================================================
# DENSE LETTER
# ============================================================

def dense_letter(
    number
):

    result = ""

    number += 1


    while number > 0:

        number -= 1


        result = (
            chr(
                ord(
                    "a"
                )
                +
                (
                    number
                    %
                    26
                )
            )
            +
            result
        )


        number //= 26


    return result


# ============================================================
# ASSIGN DENSE CODES
#
# SAME PHYSICAL SOURCE FILE = SAME PREFIX
# ============================================================

def assign_dense_codes():

    dense_log(
        "========================================"
    )

    dense_log(
        "STARTING DENSE CODE PROCESSING"
    )

    dense_log(
        "========================================"
    )


    connection = sqlite3.connect(
        SQLITE_DB
    )

    cursor = connection.cursor()


    cursor.execute(
        """
        UPDATE IMPORT_AIO

        SET DENSE_CODE = NULL
        """
    )


    cursor.execute(
        """
        SELECT
            ID,
            SOURCE_TYPE,
            SOURCE_FILE

        FROM IMPORT_AIO

        ORDER BY ID
        """
    )


    rows = cursor.fetchall()


    file_letter_indexes = {}

    file_row_numbers = {}

    next_letter_index = 0

    total_dense_codes = 0


    for (
        row_id,
        source_type,
        source_file
    ) in rows:

        if (
            source_file
            not in
            file_letter_indexes
        ):

            file_letter_indexes[
                source_file
            ] = next_letter_index


            file_row_numbers[
                source_file
            ] = 0


            dense_log(
                "NEW SOURCE FILE"
                " | "
                f"PREFIX="
                f"{dense_letter(next_letter_index)}"
                " | "
                f"FILE={source_file}"
            )


            next_letter_index += 1


        file_row_numbers[
            source_file
        ] += 1


        current_letter = dense_letter(
            file_letter_indexes[
                source_file
            ]
        )


        current_number = (
            file_row_numbers[
                source_file
            ]
        )


        dense_code = (
            f"{current_letter}"
            f"+"
            f"{current_number:04d}"
        )


        cursor.execute(
            """
            UPDATE IMPORT_AIO

            SET DENSE_CODE = ?

            WHERE ID = ?
            """,
            (
                dense_code,
                row_id
            )
        )


        dense_log(
            "DENSE CODE ASSIGNED"
            " | "
            f"ID={row_id}"
            " | "
            f"TYPE={source_type}"
            " | "
            f"FILE={source_file}"
            " | "
            f"DENSE_CODE={dense_code}"
        )


        total_dense_codes += 1


    connection.commit()

    connection.close()


    dense_log(
        "TOTAL SOURCE FILES: "
        f"{len(file_letter_indexes)}"
    )


    dense_log(
        "TOTAL DENSE CODES: "
        f"{total_dense_codes}"
    )


# ============================================================
# CREATE X_TABLE
#
# COPY ALL CSV_X ROWS
# ============================================================

def create_x_table():

    x_table_log(
        "========================================"
    )

    x_table_log(
        "STARTING X_TABLE PROCESSING"
    )

    x_table_log(
        "========================================"
    )


    connection = sqlite3.connect(
        SQLITE_DB
    )

    cursor = connection.cursor()


    cursor.execute(
        """
        DROP TABLE IF EXISTS X_TABLE
        """
    )


    cursor.execute(
        """
        CREATE TABLE X_TABLE
        (
            ID INTEGER,

            SOURCE_TYPE TEXT,

            SOURCE_FILE TEXT,

            SOURCE_ROW INTEGER,

            SECTION_NUMBER INTEGER,

            TEXT_CONTENT TEXT,

            TAGS TEXT,

            NAME_MARK TEXT,

            DESCRIPTION TEXT,

            X TEXT,

            DENSE_CODE TEXT
        )
        """
    )


    cursor.execute(
        """
        SELECT
            ID,
            SOURCE_TYPE,
            SOURCE_FILE,
            SOURCE_ROW,
            SECTION_NUMBER,
            TEXT_CONTENT,
            TAGS,
            NAME_MARK,
            DESCRIPTION,
            X,
            DENSE_CODE

        FROM IMPORT_AIO

        WHERE SOURCE_TYPE = 'CSV_X'

        ORDER BY ID
        """
    )


    rows = cursor.fetchall()


    for row in rows:

        cursor.execute(
            """
            INSERT INTO X_TABLE
            (
                ID,
                SOURCE_TYPE,
                SOURCE_FILE,
                SOURCE_ROW,
                SECTION_NUMBER,
                TEXT_CONTENT,
                TAGS,
                NAME_MARK,
                DESCRIPTION,
                X,
                DENSE_CODE
            )

            VALUES
            (
                ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
            )
            """,
            row
        )


        x_table_log(
            "ROW COPIED"
            " | "
            f"ID={row[0]}"
            " | "
            f"FILE={row[2]}"
            " | "
            f"NAME_MARK_LENGTH="
            f"{len(str(row[7] or ''))}"
            " | "
            f"DENSE_CODE={row[10]}"
        )


    connection.commit()

    connection.close()


    x_table_log(
        "SOURCE_TYPE FILTER: CSV_X"
    )


    x_table_log(
        "CSV_X ROWS FOUND: "
        f"{len(rows)}"
    )


    x_table_log(
        "ROWS COPIED TO X_TABLE: "
        f"{len(rows)}"
    )


    x_table_log(
        "FINAL X_TABLE ROW COUNT: "
        f"{len(rows)}"
    )


# ============================================================
# RECREATE EXPORT FOLDER
# ============================================================

def recreate_export_dump_folder():

    if EXPORT_DUMP_FOLDER.exists():

        shutil.rmtree(
            EXPORT_DUMP_FOLDER
        )


        export_dump_log(
            "OLD EXPORT FOLDER DELETED"
            " | "
            f"{EXPORT_DUMP_FOLDER.name}"
        )


    EXPORT_DUMP_FOLDER.mkdir(
        parents=True,
        exist_ok=True
    )


    export_dump_log(
        "EXPORT FOLDER CREATED"
        " | "
        f"{EXPORT_DUMP_FOLDER.name}"
    )


# ============================================================
# CLEAN SOURCE_FILE FOR EXPORT
#
# REMOVE:
#
# IMPORT-AIO/01-MD/
# IMPORT-AIO/02-CSV/
#
# DATABASE VALUE IS UNCHANGED
# ============================================================

def clean_source_file_for_export(
    source_file
):

    source_file = str(
        source_file
        or
        ""
    ).strip()


    source_file = source_file.replace(
        "\\",
        "/"
    )


    source_file = re.sub(
        r"^IMPORT-AIO/01-MD/",
        "",
        source_file,
        flags=re.IGNORECASE
    )


    source_file = re.sub(
        r"^IMPORT-AIO/02-CSV/",
        "",
        source_file,
        flags=re.IGNORECASE
    )


    source_file = re.sub(
        r"^IMPORT-AIO/01-MD$",
        "",
        source_file,
        flags=re.IGNORECASE
    )


    source_file = re.sub(
        r"^IMPORT-AIO/02-CSV$",
        "",
        source_file,
        flags=re.IGNORECASE
    )


    return source_file


# ============================================================
# CLEAN WINDOWS FILE NAME VALUE
# ============================================================

def clean_filename_value(
    value
):

    value = str(
        value
        or
        ""
    ).strip()


    value = (
        value
        .replace(
            "\r\n",
            EXPORT_FILENAME_SEPARATOR
        )
        .replace(
            "\n",
            EXPORT_FILENAME_SEPARATOR
        )
        .replace(
            "\r",
            EXPORT_FILENAME_SEPARATOR
        )
    )


    value = re.sub(
        r'[<>:"/\\|?*\x00-\x1F]',
        "-",
        value
    )


    value = re.sub(
        r"\s+",
        " ",
        value
    ).strip()


    value = value.rstrip(
        ". "
    )


    return value


# ============================================================
# SHORTEN VALUE FOR FILENAME ONLY
#
# DATABASE DATA REMAINS FULL
# ============================================================

def shorten_filename_value(
    value,
    max_length
):

    value = str(
        value
        or
        ""
    ).strip()


    if len(
        value
    ) <= max_length:

        return value


    return (
        value[
            :max_length
        ]
        .rstrip()
        +
        "..."
    )


# ============================================================
# FINALIZE SAFE EXPORT FILENAME
#
# ALWAYS TRY TO PRESERVE DENSE_CODE
# ============================================================

def finalize_export_filename(
    filename_without_extension,
    dense_code
):

    filename_without_extension = str(
        filename_without_extension
        or
        ""
    ).strip()


    dense_code = clean_filename_value(
        dense_code
    )


    complete_filename = (
        filename_without_extension
        +
        EXPORT_DUMP_EXTENSION
    )


    if (
        len(
            complete_filename
        )
        <=
        MAX_EXPORT_FILENAME_LENGTH
    ):

        return complete_filename


    dense_tail = (
        EXPORT_FILENAME_SEPARATOR
        +
        dense_code
        +
        EXPORT_DUMP_EXTENSION
    )


    maximum_prefix_length = (
        MAX_EXPORT_FILENAME_LENGTH
        -
        len(
            dense_tail
        )
    )


    maximum_prefix_length = max(
        10,
        maximum_prefix_length
    )


    prefix = filename_without_extension


    # --------------------------------------------------------
    # REMOVE DENSE CODE FROM END FIRST
    # SO WE CAN ADD IT BACK SAFELY
    # --------------------------------------------------------

    dense_pattern = (
        r"\s*---\s*"
        +
        re.escape(
            dense_code
        )
        +
        r"$"
    )


    prefix = re.sub(
        dense_pattern,
        "",
        prefix,
        flags=re.IGNORECASE
    )


    prefix = prefix[
        :maximum_prefix_length
    ].rstrip(
        " .-"
    )


    return (
        prefix
        +
        EXPORT_FILENAME_SEPARATOR
        +
        dense_code
        +
        EXPORT_DUMP_EXTENSION
    )


# ============================================================
# BUILD NORMAL DUMP FILENAME
#
# SOURCE_TYPE
# +
# CLEAN_SOURCE_FILE
# +
# NAME_MARK
# +
# DENSE_CODE
#
# NAME_MARK SHORTENED ONLY FOR FILENAME
# ============================================================

def build_dump_filename(
    source_type,
    source_file,
    name_mark,
    dense_code
):

    source_type = clean_filename_value(
        source_type
    )


    source_file = clean_source_file_for_export(
        source_file
    )


    source_file = source_file.replace(
        "/",
        EXPORT_FILENAME_SEPARATOR
    )


    source_file = clean_filename_value(
        source_file
    )


    name_mark = clean_filename_value(
        name_mark
    )


    name_mark = shorten_filename_value(
        name_mark,
        MAX_NAME_MARK_FILENAME_LENGTH
    )


    dense_code = clean_filename_value(
        dense_code
    )


    filename_parts = [
        source_type,
        source_file,
        name_mark,
        dense_code,
    ]


    filename_parts = [
        part

        for part in filename_parts

        if part
    ]


    filename = EXPORT_FILENAME_SEPARATOR.join(
        filename_parts
    )


    filename = re.sub(
        r"(?:\s*---\s*){2,}",
        EXPORT_FILENAME_SEPARATOR,
        filename
    )


    filename = filename.strip(
        " -"
    )


    if not filename:

        filename = (
            "EMPTY"
            +
            EXPORT_FILENAME_SEPARATOR
            +
            dense_code
        )


    return finalize_export_filename(
        filename,
        dense_code
    )


# ============================================================
# BUILD MD TAG DUMP FILENAME
#
# ONE EXTRA FILE PER TAG
# ============================================================

def build_md_tag_dump_filename(
    source_type,
    source_file,
    tag,
    dense_code
):

    source_type = clean_filename_value(
        source_type
    )


    source_file = clean_source_file_for_export(
        source_file
    )


    source_file = source_file.replace(
        "/",
        EXPORT_FILENAME_SEPARATOR
    )


    source_file = clean_filename_value(
        source_file
    )


    tag = clean_filename_value(
        tag
    )


    tag = shorten_filename_value(
        tag,
        MAX_NAME_MARK_FILENAME_LENGTH
    )


    dense_code = clean_filename_value(
        dense_code
    )


    filename_parts = [
        source_type,
        source_file,
        tag,
        dense_code,
    ]


    filename_parts = [
        part

        for part in filename_parts

        if part
    ]


    filename = EXPORT_FILENAME_SEPARATOR.join(
        filename_parts
    )


    filename = re.sub(
        r"(?:\s*---\s*){2,}",
        EXPORT_FILENAME_SEPARATOR,
        filename
    )


    filename = filename.strip(
        " -"
    )


    return finalize_export_filename(
        filename,
        dense_code
    )


# ============================================================
# SPLIT MD TAGS
# ============================================================

def split_md_tags_for_export(
    tags_text
):

    if not tags_text:

        return []


    tags = []


    for line in str(
        tags_text
    ).splitlines():

        tag = line.strip()


        if tag:

            tags.append(
                tag
            )


    return tags


# ============================================================
# WRITE DUMP FILE SAFELY
#
# ONE FAILURE DOES NOT STOP EXPORT
# ============================================================

def write_dump_file(
    output_path,
    text_content,
    row_id,
    source_type,
    source_file,
    dense_code,
    export_kind
):

    try:

        output_path.write_text(
            str(
                text_content
                or
                ""
            ),
            encoding="utf-8"
        )


        return True


    except Exception as error:

        export_dump_log(
            "EXPORT ERROR"
            " | "
            f"EXPORT_KIND={export_kind}"
            " | "
            f"ID={row_id}"
            " | "
            f"SOURCE_TYPE={source_type}"
            " | "
            f"SOURCE_FILE={source_file}"
            " | "
            f"DENSE_CODE={dense_code}"
            " | "
            f"OUTPUT={output_path.name}"
            " | "
            f"FILENAME_LENGTH={len(output_path.name)}"
            " | "
            f"ERROR="
            f"{type(error).__name__}: {error}"
        )


        return False


# ============================================================
# EXPORT DUMP FILES
# ============================================================

def export_dump_files():

    export_dump_log(
        "========================================"
    )

    export_dump_log(
        "STARTING EXPORT DUMP"
    )

    export_dump_log(
        "========================================"
    )


    recreate_export_dump_folder()


    connection = sqlite3.connect(
        SQLITE_DB
    )

    cursor = connection.cursor()


    cursor.execute(
        """
        SELECT
            ID,
            SOURCE_TYPE,
            SOURCE_FILE,
            TAGS,
            NAME_MARK,
            DENSE_CODE,
            TEXT_CONTENT

        FROM IMPORT_AIO

        ORDER BY ID
        """
    )


    rows = cursor.fetchall()


    connection.close()


    export_dump_log(
        "DATABASE ROWS FOUND: "
        f"{len(rows)}"
    )


    total_normal_exported = 0

    total_md_tag_exported = 0

    total_md_tags_found = 0

    total_failed = 0

    total_skipped = 0

    total_shortened_names = 0


    for (
        row_id,
        source_type,
        source_file,
        tags_text,
        name_mark,
        dense_code,
        text_content
    ) in rows:

        source_type = str(
            source_type
            or
            ""
        ).strip()


        source_file = str(
            source_file
            or
            ""
        ).strip()


        tags_text = str(
            tags_text
            or
            ""
        )


        name_mark = str(
            name_mark
            or
            ""
        )


        dense_code = str(
            dense_code
            or
            ""
        ).strip()


        text_content = str(
            text_content
            or
            ""
        )


        # ====================================================
        # DENSE CODE REQUIRED
        # ====================================================

        if not dense_code:

            total_skipped += 1


            export_dump_log(
                "SKIPPED - NO DENSE CODE"
                " | "
                f"ID={row_id}"
                " | "
                f"SOURCE_TYPE={source_type}"
                " | "
                f"SOURCE_FILE={source_file}"
            )


            continue


        # ====================================================
        # NORMAL EXPORT
        # ====================================================

        original_clean_name_mark = (
            clean_filename_value(
                name_mark
            )
        )


        shortened_clean_name_mark = (
            shorten_filename_value(
                original_clean_name_mark,
                MAX_NAME_MARK_FILENAME_LENGTH
            )
        )


        if (
            original_clean_name_mark
            !=
            shortened_clean_name_mark
        ):

            total_shortened_names += 1


            export_dump_log(
                "NAME_MARK SHORTENED FOR FILENAME"
                " | "
                f"ID={row_id}"
                " | "
                f"SOURCE_TYPE={source_type}"
                " | "
                f"ORIGINAL_LENGTH="
                f"{len(original_clean_name_mark)}"
                " | "
                f"FILENAME_LENGTH_VALUE="
                f"{len(shortened_clean_name_mark)}"
            )


        normal_filename = build_dump_filename(
            source_type,
            source_file,
            name_mark,
            dense_code
        )


        normal_output_path = (
            EXPORT_DUMP_FOLDER
            /
            normal_filename
        )


        normal_success = write_dump_file(
            normal_output_path,
            text_content,
            row_id,
            source_type,
            source_file,
            dense_code,
            "NORMAL"
        )


        if normal_success:

            total_normal_exported += 1


            export_dump_log(
                "DUMP EXPORTED"
                " | "
                f"ID={row_id}"
                " | "
                f"SOURCE_TYPE={source_type}"
                " | "
                f"SOURCE_FILE={source_file}"
                " | "
                f"NAME_MARK={name_mark}"
                " | "
                f"DENSE_CODE={dense_code}"
                " | "
                f"FILENAME_LENGTH={len(normal_filename)}"
                " | "
                f"OUTPUT={normal_filename}"
            )


        else:

            total_failed += 1


        # ====================================================
        # MD TAG COPIES
        # ====================================================

        if (
            source_type.upper()
            !=
            "MD"
        ):

            continue


        md_tags = split_md_tags_for_export(
            tags_text
        )


        if not md_tags:

            continue


        total_md_tags_found += len(
            md_tags
        )


        export_dump_log(
            "MD TAG EXPORT START"
            " | "
            f"ID={row_id}"
            " | "
            f"SOURCE_FILE={source_file}"
            " | "
            f"TAG_COUNT={len(md_tags)}"
        )


        for (
            tag_number,
            tag
        ) in enumerate(
            md_tags,
            start=1
        ):

            tag_filename = (
                build_md_tag_dump_filename(
                    source_type,
                    source_file,
                    tag,
                    dense_code
                )
            )


            tag_output_path = (
                EXPORT_DUMP_FOLDER
                /
                tag_filename
            )


            tag_success = write_dump_file(
                tag_output_path,
                text_content,
                row_id,
                source_type,
                source_file,
                dense_code,
                "MD_TAG"
            )


            if tag_success:

                total_md_tag_exported += 1


                export_dump_log(
                    "MD TAG DUMP EXPORTED"
                    " | "
                    f"ID={row_id}"
                    " | "
                    f"TAG_NUMBER={tag_number}"
                    " | "
                    f"TAG={tag}"
                    " | "
                    f"DENSE_CODE={dense_code}"
                    " | "
                    f"TAG_IN_CONTENT=NO"
                    " | "
                    f"FILENAME_LENGTH={len(tag_filename)}"
                    " | "
                    f"OUTPUT={tag_filename}"
                )


            else:

                total_failed += 1


    # ========================================================
    # SUMMARY
    # ========================================================

    export_dump_log(
        ""
    )


    export_dump_log(
        "========================================"
    )

    export_dump_log(
        "EXPORT DUMP SUMMARY"
    )

    export_dump_log(
        "========================================"
    )


    export_dump_log(
        "SOURCE TABLE: IMPORT_AIO"
    )


    export_dump_log(
        "MAX EXPORT FILENAME LENGTH: "
        f"{MAX_EXPORT_FILENAME_LENGTH}"
    )


    export_dump_log(
        "MAX NAME_MARK FILENAME LENGTH: "
        f"{MAX_NAME_MARK_FILENAME_LENGTH}"
    )


    export_dump_log(
        "NORMAL DUMP FILES EXPORTED: "
        f"{total_normal_exported}"
    )


    export_dump_log(
        "MD TAG VALUES FOUND: "
        f"{total_md_tags_found}"
    )


    export_dump_log(
        "ADDITIONAL MD TAG FILES EXPORTED: "
        f"{total_md_tag_exported}"
    )


    export_dump_log(
        "TOTAL SUCCESSFUL FILES: "
        f"{total_normal_exported + total_md_tag_exported}"
    )


    export_dump_log(
        "NAME_MARK FILENAMES SHORTENED: "
        f"{total_shortened_names}"
    )


    export_dump_log(
        "EXPORT FAILURES: "
        f"{total_failed}"
    )


    export_dump_log(
        "ROWS SKIPPED: "
        f"{total_skipped}"
    )


    export_dump_log(
        "TEXT FILE CONTENT SOURCE: TEXT_CONTENT"
    )


    export_dump_log(
        "NAME_MARK INCLUDED IN TEXT BODY: NO"
    )


    export_dump_log(
        "MD TAG INCLUDED IN TEXT BODY: NO"
    )


    export_dump_log(
        "========================================"
    )


# ============================================================
# DATABASE SUMMARY
# ============================================================

def database_summary():

    connection = sqlite3.connect(
        SQLITE_DB
    )

    cursor = connection.cursor()


    cursor.execute(
        """
        SELECT COUNT(*)

        FROM IMPORT_AIO
        """
    )


    total_rows = (
        cursor.fetchone()[0]
    )


    cursor.execute(
        """
        SELECT COUNT(*)

        FROM IMPORT_AIO

        WHERE SOURCE_TYPE = 'MD'
        """
    )


    md_rows = (
        cursor.fetchone()[0]
    )


    cursor.execute(
        """
        SELECT COUNT(*)

        FROM IMPORT_AIO

        WHERE SOURCE_TYPE = 'CSV'
        """
    )


    csv_rows = (
        cursor.fetchone()[0]
    )


    cursor.execute(
        """
        SELECT COUNT(*)

        FROM IMPORT_AIO

        WHERE SOURCE_TYPE = 'CSV_X'
        """
    )


    csv_x_rows = (
        cursor.fetchone()[0]
    )


    cursor.execute(
        """
        SELECT COUNT(*)

        FROM X_TABLE
        """
    )


    x_table_rows = (
        cursor.fetchone()[0]
    )


    connection.close()


    log(
        "========================================"
    )


    log(
        "DATABASE SUMMARY"
    )


    log(
        "========================================"
    )


    log(
        "IMPORT_AIO TOTAL ROWS: "
        f"{total_rows}"
    )


    log(
        "MD ROWS: "
        f"{md_rows}"
    )


    log(
        "CSV ROWS: "
        f"{csv_rows}"
    )


    log(
        "CSV_X ROWS: "
        f"{csv_x_rows}"
    )


    log(
        "X_TABLE ROWS: "
        f"{x_table_rows}"
    )


# ============================================================
# MAIN
# ============================================================

def main():

    # --------------------------------------------------------
    # LOG SYSTEM FIRST
    # --------------------------------------------------------

    recreate_log_files()


    log(
        "========================================"
    )

    log(
        "STARTING IMPORT-AIO"
    )

    log(
        "========================================"
    )


    # --------------------------------------------------------
    # IMPORT FOLDERS
    # --------------------------------------------------------

    create_import_folders()


    # --------------------------------------------------------
    # DATABASE
    # --------------------------------------------------------

    create_sqlite_database()


    # --------------------------------------------------------
    # NESTED SAMPLE CREATION
    # --------------------------------------------------------

    ensure_unused_md_samples()

    ensure_unused_csv_samples()


    # --------------------------------------------------------
    # MD
    # --------------------------------------------------------

    process_md_files()


    # --------------------------------------------------------
    # CSV
    # --------------------------------------------------------

    process_csv_files()


    # --------------------------------------------------------
    # DENSE CODES
    # --------------------------------------------------------

    assign_dense_codes()


    # --------------------------------------------------------
    # X_TABLE
    # --------------------------------------------------------

    create_x_table()


    # --------------------------------------------------------
    # EXPORT DUMP
    #
    # LONG CSV_X NAME_MARK VALUES
    # ARE SHORTENED ONLY FOR FILENAME
    #
    # DATABASE AND TEXT CONTENT
    # REMAIN COMPLETE
    # --------------------------------------------------------

    export_dump_files()


    # --------------------------------------------------------
    # SUMMARY
    # --------------------------------------------------------

    database_summary()


    # --------------------------------------------------------
    # COMPLETE
    # --------------------------------------------------------

    log(
        "========================================"
    )

    log(
        "IMPORT-AIO COMPLETED"
    )

    log(
        "========================================"
    )


    save_logs()


# ============================================================
# RUN
# ============================================================

if __name__ == "__main__":

    try:

        main()


    except Exception as error:

        try:

            log(
                "FATAL ERROR"
                " | "
                f"TYPE={type(error).__name__}"
                " | "
                f"ERROR={error}"
            )


        except Exception:

            print(
                "FATAL ERROR"
                " | "
                f"{type(error).__name__}"
                " | "
                f"{error}"
            )


        raise