monitor/core/deployer/recipeEngine.js

const modulename = 'RecipeEngine';
import { promisify } from 'node:util';
import fse from 'fs-extra';
import fsp from 'node:fs/promises';
import path from 'node:path';
import stream from 'node:stream';
import StreamZip from 'node-stream-zip';
import { cloneDeep, escapeRegExp } from 'lodash-es';
import mysql from 'mysql2/promise';
import got from '@lib/got';
import consoleFactory from '@lib/console';
const console = consoleFactory(modulename);


//Helper functions
const safePath = (base, suffix) => {
    const safeSuffix = path.normalize(suffix).replace(/^(\.\.(\/|\\|$))+/, '');
    return path.join(base, safeSuffix);
};
const isPathLinear = (pathInput) => {
    return pathInput.match(/(\.\.(\/|\\|$))+/g) === null;
};
const isPathRoot = (pathInput) => {
    return /^\.[/\\]*$/.test(pathInput);
};
const pathCleanTrail = (pathInput) => {
    return pathInput.replace(/[/\\]+$/, '');
};
const isPathValid = (pathInput, acceptRoot = true) => {
    return (
        typeof pathInput == 'string'
        && pathInput.length
        && isPathLinear(pathInput)
        && (acceptRoot || !isPathRoot(pathInput))
    );
};
const replaceVars = (inputString, deployerCtx) => {
    const allVars = Object.keys(deployerCtx);
    for (const varName of allVars) {
        const varNameReplacer = new RegExp(escapeRegExp(`{{${varName}}}`), 'g');
        inputString = inputString.replace(varNameReplacer, deployerCtx[varName].toString());
    }
    return inputString;
};


/**
 * Downloads a file to a target path using streams
 */
const validatorDownloadFile = (options) => {
    return (
        typeof options.url == 'string'
        && isPathValid(options.path)
    );
};
const taskDownloadFile = async (options, basePath, deployerCtx) => {
    if (!validatorDownloadFile(options)) throw new Error('invalid options');
    if (options.path.endsWith('/')) throw new Error('target filename not specified'); //FIXME: this should be on the validator

    //Process and create target file/path
    const destPath = safePath(basePath, options.path);
    await fse.outputFile(destPath, 'file save attempt, please ignore or remove');

    //Start file download and create write stream
    deployerCtx.$step = 'before stream';
    const gotOptions = {
        timeout: { request: 150e3 },
        retry: { limit: 5 },
    };
    const gotStream = got.stream(options.url, gotOptions);
    gotStream.on('downloadProgress', (progress) => {
        deployerCtx.$step = `downloading ${Math.round(progress.percent * 100)}%`;
    });
    const pipeline = promisify(stream.pipeline);
    await pipeline(
        gotStream,
        fse.createWriteStream(destPath),
    );
    deployerCtx.$step = 'after stream';
};


/**
 * Downloads a github repository with an optional reference (branch, tag, commit hash) or subpath.
 * If the directory structure does not exist, it is created.
 */
const githubRepoSourceRegex = /^((https?:\/\/github\.com\/)?|@)?([\w.\-_]+)\/([\w.\-_]+).*$/;
const validatorDownloadGithub = (options) => {
    return (
        typeof options.src == 'string'
        && isPathValid(options.dest, false)
        && (typeof options.ref == 'string' || typeof options.ref == 'undefined')
        && (typeof options.subpath == 'string' || typeof options.subpath == 'undefined')
    );
};
const taskDownloadGithub = async (options, basePath, deployerCtx) => {
    if (!validatorDownloadGithub(options)) throw new Error('invalid options');
    //FIXME: caso seja eperm, tentar criar um arquivo na pasta e checar se funciona

    //Parsing source
    deployerCtx.$step = 'task start';
    const srcMatch = options.src.match(githubRepoSourceRegex);
    if (!srcMatch || !srcMatch[3] || !srcMatch[4]) throw new Error('invalid repository');
    const repoOwner = srcMatch[3];
    const repoName = srcMatch[4];

    //Setting git ref
    let reference;
    if (options.ref) {
        reference = options.ref;
    } else {
        const data = await got.get(
            `https://api.github.com/repos/${repoOwner}/${repoName}`,
            {
                timeout: { request: 15e3 }
            }
        ).json();
        if (typeof data !== 'object' || !data.default_branch) {
            throw new Error('reference not set, and wasn ot able to detect using github\'s api');
        }
        reference = data.default_branch;
    }
    deployerCtx.$step = 'ref set';

    //Preparing vars
    const downURL = `https://api.github.com/repos/${repoOwner}/${repoName}/zipball/${reference}`;
    const tmpFilePath = path.join(basePath, `.${(Date.now() % 100000000).toString(36)}.download`);
    const destPath = safePath(basePath, options.dest);

    //Downloading file
    deployerCtx.$step = 'before stream';
    const gotOptions = {
        timeout: { request: 150e3 },
        retry: { limit: 5 },
    };
    const gotStream = got.stream(downURL, gotOptions);
    gotStream.on('downloadProgress', (progress) => {
        deployerCtx.$step = `downloading ${Math.round(progress.percent * 100)}%`;
    });
    const pipeline = promisify(stream.pipeline);
    await pipeline(
        gotStream,
        fse.createWriteStream(tmpFilePath),
    );
    deployerCtx.$step = 'after stream';

    //Extracting files
    const zip = new StreamZip.async({ file: tmpFilePath });
    const entries = Object.values(await zip.entries());
    if (!entries.length || !entries[0].isDirectory) throw new Error('unexpected zip structure');
    const zipSubPath = path.posix.join(entries[0].name, options.subpath || '');
    deployerCtx.$step = 'zip parsed';
    await fsp.mkdir(destPath, { recursive: true });
    deployerCtx.$step = 'dest path created';
    await zip.extract(zipSubPath, destPath);
    deployerCtx.$step = 'zip extracted';
    await zip.close();
    deployerCtx.$step = 'zip closed';

    //Removing temp path
    await fse.remove(tmpFilePath);
    deployerCtx.$step = 'task finished';
};


/**
 * Removes a file or directory. The directory can have contents. If the path does not exist, silently does nothing.
 */
const validatorRemovePath = (options) => {
    return (
        isPathValid(options.path, false)
    );
};
const taskRemovePath = async (options, basePath, deployerCtx) => {
    if (!validatorRemovePath(options)) throw new Error('invalid options');

    //Process and create target file/path
    const targetPath = safePath(basePath, options.path);

    //NOTE: being extra safe about not deleting itself
    const cleanBasePath = pathCleanTrail(path.normalize(basePath));
    if (cleanBasePath == targetPath) throw new Error('cannot remove base folder');
    await fse.remove(targetPath);
};


/**
 * Ensures that the directory exists. If the directory structure does not exist, it is created.
 */
const validatorEnsureDir = (options) => {
    return (
        isPathValid(options.path, false)
    );
};
const taskEnsureDir = async (options, basePath, deployerCtx) => {
    if (!validatorEnsureDir(options)) throw new Error('invalid options');

    //Process and create target file/path
    const destPath = safePath(basePath, options.path);
    await fse.ensureDir(destPath);
};


/**
 * Extracts a ZIP file to a targt folder.
 * NOTE: wow that was not easy to pick a library!
 *  - tar: no zip files
 *  - minizlib: terrible docs, probably too low level
 *  - yauzl: deprecation warning, slow
 *  - extract-zip: deprecation warning, slow due to yauzl
 *  - jszip: it's more a browser thing than node, doesn't appear to have an extract option
 *  - archiver: no extract
 *  - zip-stream: no extract
 *  - adm-zip: 50ms the old one, shitty
 *  - node-stream-zip: 180ms, acceptable
 *  - unzip: last update 7 years ago
 *  - unzipper: haven't tested
 *  - fflate: haven't tested
 *  - decompress-zip: haven't tested
 */
const validatorUnzip = (options) => {
    return (
        isPathValid(options.src, false)
        && isPathValid(options.dest)
    );
};
const taskUnzip = async (options, basePath, deployerCtx) => {
    if (!validatorUnzip(options)) throw new Error('invalid options');

    const srcPath = safePath(basePath, options.src);
    const destPath = safePath(basePath, options.dest);
    await fsp.mkdir(destPath, { recursive: true });

    const zip = new StreamZip.async({ file: srcPath });
    const count = await zip.extract(null, destPath);
    console.log(`Extracted ${count} entries`);
    await zip.close();
};


/**
 * Moves a file or directory. The directory can have contents.
 */
const validatorMovePath = (options) => {
    return (
        isPathValid(options.src, false)
        && isPathValid(options.dest, false)
    );
};
const taskMovePath = async (options, basePath, deployerCtx) => {
    if (!validatorMovePath(options)) throw new Error('invalid options');

    const srcPath = safePath(basePath, options.src);
    const destPath = safePath(basePath, options.dest);
    await fse.move(srcPath, destPath, {
        overwrite: (options.overwrite === 'true' || options.overwrite === true),
    });
};


/**
 * Copy a file or directory. The directory can have contents.
 * TODO: add a filter property and use a glob lib in the fse.copy filter function
 */
const validatorCopyPath = (options) => {
    return (
        isPathValid(options.src)
        && isPathValid(options.dest)
    );
};
const taskCopyPath = async (options, basePath, deployerCtx) => {
    if (!validatorCopyPath(options)) throw new Error('invalid options');

    const srcPath = safePath(basePath, options.src);
    const destPath = safePath(basePath, options.dest);
    await fse.copy(srcPath, destPath, {
        overwrite: (typeof options.overwrite !== 'undefined' && (options.overwrite === 'true' || options.overwrite === true)),
    });
};


/**
 * Writes or appends data to a file. If not in the append mode, the file will be overwritten and the directory structure will be created if it doesn't exists.
 */
const validatorWriteFile = (options) => {
    return (
        typeof options.data == 'string'
        && options.data.length
        && isPathValid(options.file, false)
    );
};
const taskWriteFile = async (options, basePath, deployerCtx) => {
    if (!validatorWriteFile(options)) throw new Error('invalid options');

    const filePath = safePath(basePath, options.file);
    if (options.append === 'true' || options.append === true) {
        await fse.appendFile(filePath, options.data);
    } else {
        await fse.outputFile(filePath, options.data);
    }
};


/**
 * Replaces a string in the target file or files array based on a search string.
 * Modes:
 *  - template: (default) target string will be processed for vars
 *  - literal: normal string search/replace without any vars
 *  - all_vars: all vars.toString() will be replaced. The search option will be ignored
 */
const validatorReplaceString = (options) => {
    //Validate file
    const fileList = (Array.isArray(options.file)) ? options.file : [options.file];
    if (fileList.some((s) => !isPathValid(s, false))) {
        return false;
    }

    //Validate mode
    if (
        typeof options.mode == 'undefined'
        || options.mode == 'template'
        || options.mode == 'literal'
    ) {
        return (
            typeof options.search == 'string'
            && options.search.length
            && typeof options.replace == 'string'
        );
    } else if (options.mode == 'all_vars') {
        return true;
    } else {
        return false;
    }
};
const taskReplaceString = async (options, basePath, deployerCtx) => {
    if (!validatorReplaceString(options)) throw new Error('invalid options');

    const fileList = (Array.isArray(options.file)) ? options.file : [options.file];
    for (let i = 0; i < fileList.length; i++) {
        const filePath = safePath(basePath, fileList[i]);
        const original = await fse.readFile(filePath, 'utf8');
        let changed;
        if (typeof options.mode == 'undefined' || options.mode == 'template') {
            changed = original.replace(new RegExp(options.search, 'g'), replaceVars(options.replace, deployerCtx));
        } else if (options.mode == 'all_vars') {
            changed = replaceVars(original, deployerCtx);
        } else if (options.mode == 'literal') {
            changed = original.replace(new RegExp(options.search, 'g'), options.replace);
        }
        await fse.writeFile(filePath, changed);
    }
};


/**
 * Connects to a MySQL/MariaDB server and creates a database if the dbName variable is null.
 */
const validatorConnectDatabase = (options) => {
    return true;
};
const taskConnectDatabase = async (options, basePath, deployerCtx) => {
    if (!validatorConnectDatabase(options)) throw new Error('invalid options');
    if (typeof deployerCtx.dbHost !== 'string') throw new Error('invalid dbHost');
    if (typeof deployerCtx.dbPort !== 'number') throw new Error('invalid dbPort, should be number');
    if (typeof deployerCtx.dbUsername !== 'string') throw new Error('invalid dbUsername');
    if (typeof deployerCtx.dbPassword !== 'string') throw new Error('dbPassword should be a string');
    if (typeof deployerCtx.dbName !== 'string') throw new Error('dbName should be a string');
    if (typeof deployerCtx.dbDelete !== 'boolean') throw new Error('dbDelete should be a boolean');
    //Connect to the database
    const mysqlOptions = {
        host: deployerCtx.dbHost,
        port: deployerCtx.dbPort,
        user: deployerCtx.dbUsername,
        password: deployerCtx.dbPassword,
        multipleStatements: true,
    };
    deployerCtx.dbConnection = await mysql.createConnection(mysqlOptions);
    const escapedDBName = mysql.escapeId(deployerCtx.dbName);
    if (deployerCtx.dbDelete) {
        await deployerCtx.dbConnection.query(`DROP DATABASE IF EXISTS ${escapedDBName}`);
    }
    await deployerCtx.dbConnection.query(`CREATE DATABASE IF NOT EXISTS ${escapedDBName} CHARACTER SET utf8 COLLATE utf8_general_ci`);
    await deployerCtx.dbConnection.query(`USE ${escapedDBName}`);
};


/**
 * Runs a SQL query in the previously connected database. This query can be a file path or a string.
 */
const validatorQueryDatabase = (options) => {
    if (typeof options.file !== 'undefined' && typeof options.query !== 'undefined') return false;
    if (typeof options.file == 'string') return isPathValid(options.file, false);
    if (typeof options.query == 'string') return options.query.length;
    return false;
};
const taskQueryDatabase = async (options, basePath, deployerCtx) => {
    if (!validatorQueryDatabase(options)) throw new Error('invalid options');
    if (!deployerCtx.dbConnection) throw new Error('Database connection not found. Run connect_database before query_database');

    let sql;
    if (options.file) {
        const filePath = safePath(basePath, options.file);
        sql = await fse.readFile(filePath, 'utf8');
    } else {
        sql = options.query;
    }
    await deployerCtx.dbConnection.query(sql);
};


/**
 * Loads variables from a json file to the context.
 */
const validatorLoadVars = (options) => {
    return isPathValid(options.src, false);
};
const taskLoadVars = async (options, basePath, deployerCtx) => {
    if (!validatorLoadVars(options)) throw new Error('invalid options');

    const srcPath = safePath(basePath, options.src);
    const rawData = await fse.readFile(srcPath, 'utf8');
    const inData = JSON.parse(rawData);
    inData.dbConnection = undefined;
    Object.assign(deployerCtx, inData);
};


/**
 * DEBUG Just wastes time /shrug
 */
const validatorWasteTime = (options) => {
    return (typeof options.seconds == 'number');
};
const taskWasteTime = (options, basePath, deployerCtx) => {
    return new Promise((resolve, reject) => {
        setTimeout(() => {
            resolve(true);
        }, options.seconds * 1000);
    });
};


/**
 * DEBUG Fail fail fail :o
 */
const taskFailTest = async (options, basePath, deployerCtx) => {
    throw new Error('test error :p');
};


/**
 * DEBUG logs all ctx vars
 */
const taskDumpVars = async (options, basePath, deployerCtx) => {
    const toDump = cloneDeep(deployerCtx);
    toDump.dbConnection = toDump?.dbConnection?.constructor?.name;
    console.dir(toDump);
};


/*
DONE:
    - download_file
    - remove_path (file or folder)
    - ensure_dir
    - unzip
    - move_path (file or folder)
    - copy_path (file or folder)
    - write_file (with option to append only)
    - replace_string (single or array)
    - connect_database (connects to mysql, creates db if not set)
    - query_database (file or string)
    - download_github (with ref and subpath)
    - load_vars

DEBUG:
    - waste_time
    - fail_test
    - dump_vars

TODO:
    - ??????
*/


//Exports
export default {
    download_file: {
        validate: validatorDownloadFile,
        run: taskDownloadFile,
        timeoutSeconds: 180,
    },
    download_github: {
        validate: validatorDownloadGithub,
        run: taskDownloadGithub,
        timeoutSeconds: 180,
    },
    remove_path: {
        validate: validatorRemovePath,
        run: taskRemovePath,
        timeoutSeconds: 15,
    },
    ensure_dir: {
        validate: validatorEnsureDir,
        run: taskEnsureDir,
        timeoutSeconds: 15,
    },
    unzip: {
        validate: validatorUnzip,
        run: taskUnzip,
        timeoutSeconds: 180,
    },
    move_path: {
        validate: validatorMovePath,
        run: taskMovePath,
        timeoutSeconds: 180,
    },
    copy_path: {
        validate: validatorCopyPath,
        run: taskCopyPath,
        timeoutSeconds: 180,
    },
    write_file: {
        validate: validatorWriteFile,
        run: taskWriteFile,
        timeoutSeconds: 15,
    },
    replace_string: {
        validate: validatorReplaceString,
        run: taskReplaceString,
        timeoutSeconds: 15,
    },
    connect_database: {
        validate: validatorConnectDatabase,
        run: taskConnectDatabase,
        timeoutSeconds: 30,
    },
    query_database: {
        validate: validatorQueryDatabase,
        run: taskQueryDatabase,
        timeoutSeconds: 90,
    },
    load_vars: {
        validate: validatorLoadVars,
        run: taskLoadVars,
        timeoutSeconds: 5,
    },

    //DEBUG only
    waste_time: {
        validate: validatorWasteTime,
        run: taskWasteTime,
        timeoutSeconds: 300,
    },
    fail_test: {
        validate: (() => true),
        run: taskFailTest,
        timeoutSeconds: 300,
    },
    dump_vars: {
        validate: (() => true),
        run: taskDumpVars,
        timeoutSeconds: 5,
    },
};