diff --git a/.github/logToCs.py b/.github/logToCs.py new file mode 100755 index 00000000000..2dd5d0ed0e3 --- /dev/null +++ b/.github/logToCs.py @@ -0,0 +1,382 @@ +#!/usr/bin/env python3 +# pylint: disable=invalid-name +""" +Convert a log to CheckStyle format. + +Url: https://github.com/mdeweerd/LogToCheckStyle + +The log can then be used for generating annotations in a github action. + +Note: this script is very young and "quick and dirty". + Patterns can be added to "PATTERNS" to match more messages. + +# Examples + +Assumes that logToCs.py is available as .github/logToCs.py. + +## Example 1: + + +```yaml + - run: | + pre-commit run -all-files | tee pre-commit.log + .github/logToCs.py pre-commit.log pre-commit.xml + - uses: staabm/annotate-pull-request-from-checkstyle-action@v1 + with: + files: pre-commit.xml + notices-as-warnings: true # optional +``` + +## Example 2: + + +```yaml + - run: | + pre-commit run --all-files | tee pre-commit.log + - name: Add results to PR + if: ${{ always() }} + run: | + .github/logToCs.py pre-commit.log | cs2pr +``` + +Author(s): + - https://github.com/mdeweerd + +License: MIT License + +""" + +import argparse +import os +import re +import sys +import xml.etree.ElementTree as ET # nosec + + +def remove_prefix(string, prefix): + """ + Remove prefix from string + + Provided for backward compatibility. + """ + if prefix and string.startswith(prefix): + return string[len(prefix) :] + return string + + +def convert_to_checkstyle(messages, root_path=None): + """ + Convert provided message to CheckStyle format. + """ + root = ET.Element("checkstyle") + for message in messages: + fields = parse_message(message) + if fields: + add_error_entry(root, **fields, root_path=root_path) + return ET.tostring(root, encoding="utf_8").decode("utf_8") + + +def convert_text_to_checkstyle(text, root_path=None): + """ + Convert provided message to CheckStyle format. + """ + root = ET.Element("checkstyle") + for fields in parse_file(text): + if fields: + add_error_entry(root, **fields, root_path=root_path) + return ET.tostring(root, encoding="utf_8").decode("utf_8") + + +ANY_REGEX = r".*?" +FILE_REGEX = r"\s*(?P\S.*?)\s*?" +EOL_REGEX = r"[\r\n]" +LINE_REGEX = r"\s*(?P\d+?)\s*?" +COLUMN_REGEX = r"\s*(?P\d+?)\s*?" +SEVERITY_REGEX = r"\s*(?Perror|warning|notice|style|info)\s*?" +MSG_REGEX = r"\s*(?P.+?)\s*?" +MULTILINE_MSG_REGEX = r"\s*(?P(?:.|.[\r\n])+)" +# cpplint confidence index +CONFIDENCE_REGEX = r"\s*\[(?P\d+)\]\s*?" + + +# List of message patterns, add more specific patterns earlier in the list +# Creating patterns by using constants makes them easier to define and read. +PATTERNS = [ + # beautysh + # File ftp.sh: error: "esac" before "case" in line 90. + re.compile( + f"^File {FILE_REGEX}:{SEVERITY_REGEX}:" + f" {MSG_REGEX} in line {LINE_REGEX}.$" + ), + # beautysh + # File socks4echo.sh: error: indent/outdent mismatch: -2. + re.compile(f"^File {FILE_REGEX}:{SEVERITY_REGEX}: {MSG_REGEX}$"), + # ESLint (JavaScript Linter), RoboCop, shellcheck + # path/to/file.js:10:2: Some linting issue + # path/to/file.rb:10:5: Style/Indentation: Incorrect indentation detected + # path/to/script.sh:10:1: SC2034: Some shell script issue + re.compile(f"^{FILE_REGEX}:{LINE_REGEX}:{COLUMN_REGEX}: {MSG_REGEX}$"), + # Cpplint default output: + # '%s:%s: %s [%s] [%d]\n' + # % (filename, linenum, message, category, confidence) + re.compile(f"^{FILE_REGEX}:{LINE_REGEX}:{MSG_REGEX}{CONFIDENCE_REGEX}$"), + # MSVC + # file.cpp(10): error C1234: Some error message + re.compile( + f"^{FILE_REGEX}\\({LINE_REGEX}\\):{SEVERITY_REGEX}{MSG_REGEX}$" + ), + # Java compiler + # File.java:10: error: Some error message + re.compile(f"^{FILE_REGEX}:{LINE_REGEX}:{SEVERITY_REGEX}:{MSG_REGEX}$"), + # Python + # File ".../logToCs.py", line 90 (note: code line follows) + re.compile(f'^File "{FILE_REGEX}", line {LINE_REGEX}$'), + # Pylint, others + # path/to/file.py:10: [C0111] Missing docstring + # others + re.compile(f"^{FILE_REGEX}:{LINE_REGEX}: {MSG_REGEX}$"), + # Shellcheck: + # In script.sh line 76: + re.compile( + f"^In {FILE_REGEX} line {LINE_REGEX}:{EOL_REGEX}?" + f"({MULTILINE_MSG_REGEX})?{EOL_REGEX}{EOL_REGEX}" + ), + # eslint: + # /path/to/filename + # 14:5 error Unexpected trailing comma comma-dangle + re.compile( + f"^{FILE_REGEX}{EOL_REGEX}" + rf"\s+{LINE_REGEX}:{COLUMN_REGEX}\s+{SEVERITY_REGEX}\s+{MSG_REGEX}$" + ), +] + +# Severities available in CodeSniffer report format +SEVERITY_NOTICE = "notice" +SEVERITY_WARNING = "warning" +SEVERITY_ERROR = "error" + + +def strip_ansi(text: str): + """ + Strip ANSI escape sequences from string (colors, etc) + """ + return re.sub(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", text) + + +def parse_file(text): + """ + Parse all messages in a file + + Returns the fields in a dict. + """ + # regex required to allow same group names + try: + import regex # pylint: disable=import-outside-toplevel + except ImportError as exc: + raise ImportError( + "The 'parsefile' method requires 'python -m pip install regex'" + ) from exc + + patterns = [pattern.pattern for pattern in PATTERNS] + # patterns = [PATTERNS[0].pattern] + + full_regex = "(?:(?:" + (")|(?:".join(patterns)) + "))" + results = [] + + for fields in regex.finditer( + full_regex, strip_ansi(text), regex.MULTILINE + ): + if not fields: + continue + result = fields.groupdict() + + if len(result) == 0: + continue + severity = result.get("severity", None) + confidence = result.pop("confidence", None) + + if confidence is not None: + # Convert confidence level of cpplint + # to warning, etc. + confidence = int(confidence) + + if confidence <= 1: + severity = SEVERITY_NOTICE + elif confidence >= 5: + severity = SEVERITY_ERROR + else: + severity = SEVERITY_WARNING + + if severity is None: + severity = SEVERITY_ERROR + else: + severity = severity.lower() + + if severity in ["info", "style"]: + severity = SEVERITY_NOTICE + + result["severity"] = severity + + results.append(result) + + return results + + +def parse_message(message): + """ + Parse message until it matches a pattern. + + Returns the fields in a dict. + """ + for pattern in PATTERNS: + fields = pattern.match(message) + if not fields: + continue + result = fields.groupdict() + if len(result) == 0: + continue + + if "confidence" in result: + # Convert confidence level of cpplint + # to warning, etc. + confidence = int(result["confidence"]) + del result["confidence"] + + if confidence <= 1: + severity = SEVERITY_NOTICE + elif confidence >= 5: + severity = SEVERITY_ERROR + else: + severity = SEVERITY_WARNING + result["severity"] = severity + + if "severity" not in result: + result["severity"] = SEVERITY_ERROR + else: + result["severity"] = result["severity"].lower() + + if result["severity"] in ["info", "style"]: + result["severity"] = SEVERITY_NOTICE + + return result + + # Nothing matched + return None + + +def add_error_entry( # pylint: disable=too-many-arguments + root, + severity, + file_name, + line=None, + column=None, + message=None, + source=None, + root_path=None, +): + """ + Add error information to the CheckStyle output being created. + """ + file_element = find_or_create_file_element( + root, file_name, root_path=root_path + ) + error_element = ET.SubElement(file_element, "error") + error_element.set("severity", severity) + if line: + error_element.set("line", line) + if column: + error_element.set("column", column) + if message: + error_element.set("message", message) + if source: + # To verify if this is a valid attribute + error_element.set("source", source) + + +def find_or_create_file_element(root, file_name: str, root_path=None): + """ + Find/create file element in XML document tree. + """ + + if root_path is not None: + file_name = remove_prefix(file_name, root_path) + for file_element in root.findall("file"): + if file_element.get("name") == file_name: + return file_element + file_element = ET.SubElement(root, "file") + file_element.set("name", file_name) + return file_element + + +def main(): + """ + Parse the script arguments and get the conversion done. + """ + parser = argparse.ArgumentParser( + description="Convert messages to Checkstyle XML format." + ) + parser.add_argument( + "input", help="Input file. Use '-' for stdin.", nargs="?", default="-" + ) + parser.add_argument( + "output", + help="Output file. Use '-' for stdout.", + nargs="?", + default="-", + ) + parser.add_argument( + "-i", + "--in", + dest="input_named", + help="Input filename. Overrides positional input.", + ) + parser.add_argument( + "-o", + "--out", + dest="output_named", + help="Output filename. Overrides positional output.", + ) + parser.add_argument( + "--root", + metavar="ROOT_PATH", + help="Root directory to remove from file paths." + " Defaults to working directory.", + default=os.getcwd(), + ) + + args = parser.parse_args() + + if args.input == "-" and args.input_named: + with open( + args.input_named, encoding="utf_8", errors="surrogateescape" + ) as input_file: + text = input_file.read() + elif args.input != "-": + with open( + args.input, encoding="utf_8", errors="surrogateescape" + ) as input_file: + text = input_file.read() + else: + text = sys.stdin.read() + + root_path = os.path.join(args.root, "") + + try: + checkstyle_xml = convert_text_to_checkstyle(text, root_path=root_path) + except ImportError: + checkstyle_xml = convert_to_checkstyle( + re.split(r"[\r\n]+", text), root_path=root_path + ) + + if args.output == "-" and args.output_named: + with open(args.output_named, "w", encoding="utf_8") as output_file: + output_file.write(checkstyle_xml) + elif args.output != "-": + with open(args.output, "w", encoding="utf_8") as output_file: + output_file.write(checkstyle_xml) + else: + print(checkstyle_xml) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 00000000000..9ade67aee82 --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,64 @@ +--- +name: pre-commit +on: + pull_request: + push: +jobs: + pre-commit: + runs-on: ubuntu-latest + env: + LOG_TO_CS: .github/logToCs.py + RAW_LOG: pre-commit.log + CS_XML: pre-commit.xml + steps: + - name: Install required tools + run: sudo apt-get update && sudo apt-get install cppcheck + if: false + - uses: actions/checkout@v4 + - name: Create requirements.txt if no requirements.txt or pyproject.toml + run: |- + [ -r requirements.txt ] || [ -r pyproject.toml ] || touch requirements.txt + - uses: actions/setup-python@v4 + with: + cache: pip + python-version: '3.11' + - run: python -m pip install pre-commit regex + - uses: actions/cache/restore@v3 + with: + path: ~/.cache/pre-commit/ + key: pre-commit-4|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') + }} + - name: Run pre-commit hooks + env: + # SKIP is used by pre-commit to not execute certain hooks + SKIP: php-cs,php-cbf,trailing-whitespace,end-of-file-fixer + run: | + set -o pipefail + pre-commit gc + pre-commit run --show-diff-on-failure --color=always --all-files | tee ${RAW_LOG} + - name: Convert Raw Log to CheckStyle format + if: ${{ failure() }} + run: | + python ${LOG_TO_CS} ${RAW_LOG} ${CS_XML} + - name: Annotate Source Code with Messages + uses: staabm/annotate-pull-request-from-checkstyle-action@v1 + if: ${{ failure() }} + with: + files: ${{ env.CS_XML }} + notices-as-warnings: true # optional + prepend-filename: true # optional + - uses: actions/cache/save@v3 + if: ${{ always() }} + with: + path: ~/.cache/pre-commit/ + key: pre-commit-4|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') + }} + - name: Provide log as artifact + uses: actions/upload-artifact@v3 + if: ${{ always() }} + with: + name: precommit-logs + path: | + ${{ env.RAW_LOG }} + ${{ env.CS_XML }} + retention-days: 2 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000000..8a3e5e4dbf1 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,104 @@ +--- +exclude: (?x)^( htdocs/includes/ckeditor/.* ) +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: no-commit-to-branch + args: [--branch, develop, --pattern, \d+.0] + - id: check-yaml + args: [--unsafe] + - id: check-json + - id: mixed-line-ending + exclude: (?x)^(htdocs/includes/tecnickcom/tcpdf/fonts/.*)$ + - id: trailing-whitespace + exclude_types: [markdown] + - id: end-of-file-fixer + types: [yaml] + - id: check-merge-conflict + - id: check-executables-have-shebangs + - id: check-shebang-scripts-are-executable + exclude: (?x)^( dev/tools/dolibarr-postgres2mysql.php |test/other/test_serialize.php + |test/phpunit/textutf8.txt |test/phpunit/textiso.txt |htdocs/includes/.* + |htdocs/modulebuilder/template/.* |build/debian/dolibarr.postrm |build/debian/dolibarr.postinst + |build/debian/dolibarr.config )$ + - id: fix-byte-order-marker + - id: check-case-conflict + - id: check-toml + - repo: https://github.com/lovesegfault/beautysh.git + rev: v6.2.1 + hooks: + - id: beautysh + exclude: (?x)^(dev/setup/git/hooks/pre-commit)$ + args: [--tab] + - repo: local + hooks: + - id: local-precommit-script + name: Run local script before commit if it exists + language: system + entry: bash -c '[ ! -x local.sh ] || ./local.sh' + pass_filenames: false + - repo: https://github.com/bolovsky/pre-commit-php + rev: 1.5.1 + hooks: + - id: php-cbf + files: \.(php)$ + args: [-p] + - id: php-cs + - id: php-lint + - repo: https://github.com/pre-commit/mirrors-prettier + rev: v3.0.3 + hooks: + - id: prettier + stages: [manual] + exclude: (?x)^( .*\.(phar |min\.css |lock) |htdocs/(includes|theme/common)/.* + )$ + exclude_types: + - php + - executable + - binary + - shell + - javascript + - markdown + - html + - less + - plain-text + - scss + - css + - yaml + - repo: https://github.com/adrienverge/yamllint.git + rev: v1.32.0 + hooks: + - id: yamllint + args: + - --no-warnings + - -d + - '{extends: relaxed, rules: {line-length: {max: 120}}}' + - repo: https://github.com/codespell-project/codespell + rev: v2.2.5 + hooks: + - id: codespell + # Due to a current limitation of configuration files, + # we can specify two dicts only on the CLI. + args: [-D, '-', -D, dev/tools/codespell/codespell-dict.txt] + exclude: (?x)^(.phan/stubs/.*)$ + additional_dependencies: [tomli] + - alias: codespell-lang-en_US + # Only for translations with specialised exceptions + id: codespell + files: ^htdocs/langs/en_US/.*$ + args: + - -D + - '-' + - -D + - dev/tools/codespell/codespell-dict.txt + - -L + - informations,medias,uptodate,reenable,crypted,developpers + - -L + - "creat,unitl,alltime,datas,referers,process'" + + - repo: https://github.com/shellcheck-py/shellcheck-py + rev: v0.9.0.5 + hooks: + - id: shellcheck + args: [-W, '100'] diff --git a/dev/tools/codespell/addCodespellIgnores.sh b/dev/tools/codespell/addCodespellIgnores.sh new file mode 100755 index 00000000000..0ed32973824 --- /dev/null +++ b/dev/tools/codespell/addCodespellIgnores.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# +# Script to add codespell exceptions to the ignores lines file. +# +# The file is named '...-lines-ignore' to make TAB expansion on the cli easier. +# +# The line in the ignore file must match the line in the source +# exactly. +# +# To clean up or create the ignored lines file, just do +# ```shell +# echo > dev/tools/codespell/codespell-lines-ignore.txt +# ``` +# and then execute this script +# +# author: https://github.com/mdeweerd + +codespell_ignore_file=dev/tools/codespell/codespell-lines-ignore.txt +if [ -z "${0##*.sh}" ] ; then + # Suppose running from inside script + # Get real path + script=$(realpath "$(test -L "$0" && readlink "$0" || echo "$0")") + PROJECT_ROOT=$(realpath "${script}") + + while [ "${PROJECT_ROOT}" != "/" ] ; do + [ -r "${PROJECT_ROOT}/${codespell_ignore_file}" ] && break + PROJECT_ROOT=$(dirname "${PROJECT_ROOT}") + done + if [ "${PROJECT_ROOT}" == "" ] ; then + echo "Project root not found from '${script}'" + exit 1 + fi + codespell_ignore_file=${PROJECT_ROOT}/${codespell_ignore_file} +fi + + +# Make sure we are at the root of the project +[ -r "${codespell_ignore_file}" ] || { echo "${codespell_ignore_file} not found" ; exit 1 ; } +# Then: +# - Run codespell; +# - For each line, create a grep command to find the lines; +# - Execute that command by evaluation +codespell . | perl -p -e 's@^(.*?):\d+:\s(\S+)\s.*@grep -P '"'"'\\b$2\\b'"'"' "$1" >> '"${codespell_ignore_file}"'@;' | \ + while read -r line ; do eval "$line" ; done + +# Finally, sort and remove duplicates to make merges easier. +sort -u -o "${codespell_ignore_file}"{,} + diff --git a/dev/tools/codespell/codespell-dict.txt b/dev/tools/codespell/codespell-dict.txt new file mode 100644 index 00000000000..8d270541fb1 --- /dev/null +++ b/dev/tools/codespell/codespell-dict.txt @@ -0,0 +1,8 @@ +dolibar->dolibarr +dollibar->dolibarr +dollibarr->dolibarr +not de passe->password +mot de passe->password +choosed->chosen +tableau de bord->state board +#DoliDB->DoliDB diff --git a/dev/tools/codespell/codespell-ignore.txt b/dev/tools/codespell/codespell-ignore.txt new file mode 100644 index 00000000000..92155095f18 --- /dev/null +++ b/dev/tools/codespell/codespell-ignore.txt @@ -0,0 +1,91 @@ +# List of words codespell will ignore +# one per line, case-sensitive (when not lowercase) +# PROVid +provid +# PostgreSQL +postgresql + +alltime +ba +blacklist +whitelist +bu +captial +categorie +categories +crypted +clos +contaxt +courant +datea +datee +errorstring +exten +falsy +master +medias +noe +NOO +noo +od +nd +udate +periode +projet +referer +referers +scrit +ser +slave +savvy +# Inside email +suport +te +technic +thead +udo +ue +ro +ws +# Code string +ect +tempdate +# checkES +checkes +sav +files' +# Used as array ke +seeked +# Used as translation key +developpers +# Used as var +pice +# Used as key +marge +# htdocs/projet/activity/permonth.php +tweek +# moral (var name) +mor +# reyear, remonth, reday +reday +# Strings used as keys for translation +uptodate +reenable +# Function - rename to devalidate ? +unvalidate +# Some french strings +somme +caracteres +cas +sur +Datas +datas +valide +raison +que +dur +fonction +espace +methode +# Proper names +tim diff --git a/dev/tools/codespell/codespell-lines-ignore.txt b/dev/tools/codespell/codespell-lines-ignore.txt new file mode 100644 index 00000000000..2f1cfff2ccc --- /dev/null +++ b/dev/tools/codespell/codespell-lines-ignore.txt @@ -0,0 +1,64 @@ + + print 'id.'">'; + $reponsesadd = str_split($obj->reponses); + $sql .= " SET reponses = '".$db->escape($reponsesadd)."'"; + $sql .= " SET reponses = '0".$db->escape($obj->reponses)."'"; + print ''; + GETPOST("mouvement", 'int'), + jQuery("#mouvement option").removeAttr("selected").change(); + jQuery("#mouvement option[value=0]").attr("selected","selected").trigger("change"); + jQuery("#mouvement option[value=1]").attr("selected","selected").trigger("change"); + jQuery("#mouvement").trigger("change"); + $action = 'transfert'; + $this->category->childs[] = $this->_cleanObjectDatas($cat); + $tmp = array('id_users'=>$obj->id_users, 'nom'=>$obj->name, 'reponses'=>$obj->reponses); + //si les reponses ne concerne pas la colonne effacée, on concatenate + GETPOST("mouvement", "int"), + GETPOST("mouvement", 'alpha'), + GETPOST("mouvement", 'int'), + if (jQuery("#mouvement").val() == \'0\') jQuery("#unitprice").removeAttr("disabled"); + print ''.$langs->trans("TransferStock").''; + $action = 'transfert'; + $ensemblereponses = $obj->reponses; + $sql = 'INSERT INTO '.MAIN_DB_PREFIX.'opensurvey_user_studs (nom, id_sondage, reponses, date_creation)'; + $sql = 'INSERT INTO '.MAIN_DB_PREFIX.'opensurvey_user_studs (nom, id_sondage, reponses, ip, date_creation)'; + $sql = 'SELECT s.reponses'; + $sql2 .= " SET reponses = '".$db->escape($newcar)."'"; + $this->category->childs = array(); + // mise a jour des reponses utilisateurs dans la base + if ($user->hasRight('stock', 'mouvement', 'lire')) { + jQuery("#mouvement").change(function() { + print ''.$langs->trans("TransferStock").''; + $action = 'transfert'; + $ensemblereponses = $obj->reponses; + $sql = "SELECT id_users, nom as name, id_sondage, reponses"; + $sql = "SELECT id_users, nom as name, reponses"; + $test = '/javas:cript/google.com'; + $test=""; // Same + if ($user->hasRight('stock', 'mouvement', 'creer')) { + $ensemblereponses = $obj->reponses; + $opensurveysondage->mail_admin = $_SESSION['adresse']; + $pdf->SetXY($savx, $savy); + $savy = $pdf->getY(); + $somethingshown = $formactions->showactions($object, 'mouvement', 0, 1, '', $MAXEVENT, '', $morehtmlcenter); // Show all action for product + $sql .= " SET reponses = '".$db->escape($nouveauchoix)."'"; + TaskItem(pID, pName, pStart, pEnd, pColor, pLink, pMile, pRes, pComp, pGroup, pParent, pOpen, pDepend, pCaption, pNotes, pGantt)

+ if ($action == "transfert") { + print ''; + print '