diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 47ffeb5e28f..93d43bc9d40 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,11 +7,19 @@ repos: hooks: - id: no-commit-to-branch args: [--branch, develop, --pattern, \d+.0] + - id: check-xml + exclude: | + (?x)^(htdocs/includes/.*)$ - id: check-yaml args: [--unsafe] - id: check-json - id: mixed-line-ending - exclude: (?x)^(htdocs/includes/tecnickcom/tcpdf/fonts/.*)$ + # alternative for dev/tools/fixdosfiles.sh + exclude: | + (?x)^(htdocs/includes/tecnickcom/tcpdf/fonts/.* + |.*/CRLF.*.php # Files in swiftmailer + )$ + args: [--fix=lf] - id: trailing-whitespace exclude_types: [markdown] - id: end-of-file-fixer @@ -41,8 +49,8 @@ repos: # ```shell # #!/bin/bash # MYDIR=$(dirname "$0") - # CHANGED_INTERNALS=$(git diff --name-only | grep -v includes) - # "$MYDIR/dev/tools/updatelicense.php" $CHANGED_INTERNALS + # git diff HEAD --name-only | grep -v includes | \ + # xargs "$MYDIR/dev/tools/updatelicense.php" # ``` - repo: local hooks: @@ -51,6 +59,30 @@ repos: language: system entry: bash -c '[ ! -x local.sh ] || ./local.sh' pass_filenames: false + - id: duplicate-lang-lines + stages: [manual] + name: Find duplicate language lines + files: (?x)^(htdocs/langs/en_US/.*\.lang) + language: script + entry: ./dev/tools/fixduplicatelanglines.sh + pass_filenames: false + args: [list] + - id: duplicate-lang-keys + stages: [manual] + name: Find duplicate language keys + files: (?x)^(htdocs/langs/en_US/.*\.lang) + language: script + entry: ./dev/tools/fixduplicatelangkey.sh + pass_filenames: false + args: [list] + - id: fix-alt-languages + stages: [manual] + name: Fix alt languages + # Selection: see fixaltlanguages.sh script + files: (?x)^(htdocs/langs/(e[lnstu]|k[akmno]|s[lqrv]|b[nrs]|c[asy]|n[bel]|[ip]t|a[mr]|d[ae]|f[ar]|h[ei]|m[sy]|t[ag]|u[kr]|gl|ja|lo|ru|vi|zh)_[^/]*/.*\.lang) + language: script + entry: ./dev/tools/fixaltlanguages_pre-commit.sh + pass_filenames: true # Check PHP syntax - repo: https://github.com/mdeweerd/pre-commit-php diff --git a/dev/tools/fixaltlanguages.sh b/dev/tools/fixaltlanguages.sh index 8d1092744b7..0407ff4cd6b 100755 --- a/dev/tools/fixaltlanguages.sh +++ b/dev/tools/fixaltlanguages.sh @@ -1,87 +1,115 @@ -#!/bin/sh +#!/bin/bash # Recursively deduplicate file lines on a per file basis # Useful to deduplicate language files # # Needs awk 4.0 for the inplace fixing command # -# Raphaël Doursenaud - rdoursenaud@gpcsolutions.fr - -# shellcheck disable=2006,2027,2044,2045,2046,2086,2155,2166,2268 +# Copyright (C) 2016 Raphaël Doursenaud +# Copyright (C) 2024 MDW # Syntax -if [ "x$1" != "xlist" -a "x$1" != "xfix" ] +if [ "$1" != "list" ] && [ "$1" != "fix" ] then - echo "Scan alternate language files and remove entries found into parent file" - echo "Usage: fixaltlanguages.sh (list|fix) (all|file.lang) [xx_XX]" - exit + echo "Scan alternate language files and remove entries found in parent file" + echo "Usage: fixaltlanguages.sh (list|fix) (all|file.lang) [ll_CC]" + exit 1 fi -if [ "x$2" = "x" ] +if [ "$2" = "" ] then - echo "Scan alternate language files and remove entries found into parent file" - echo "Usage: fixaltlanguages.sh (list|fix) (all|file.lang) [xx_XX]" - exit + echo "Scan alternate language files and remove entries found in parent file" + echo "Usage: fixaltlanguages.sh (list|fix) (all|file.lang) [ll_CC]" + exit 1 +fi + +ACTION=$1 +LANGFILE=$2 +LANG=$3 +exit_code=0 + + +if [ -r "$LANGFILE" ] ; then + if [ "$LANG" = "" ] ; then + LANG=$(basename "$(dirname "$LANGFILE")") + fi + LANGFILE=$(basename "$LANGFILE") fi # To detect -if [ "x$1" = "xlist" ] +if [ "$ACTION" = "list" ] then echo Feature not available + exit_code=1 fi +echo "$ACTION $LANGFILE $LANG" + # To fix -if [ "x$1" = "xfix" ] +if [ "$ACTION" = "fix" ] then - for dir in `find htdocs/langs/$3* -type d` + for dir in htdocs/langs/"$LANG"*/ do - dirshort=`basename $dir` + dirshort=$(basename "$dir") - #echo $dirshort + # echo $dirshort - export aa=`echo $dirshort | nawk -F"_" '{ print $1 }'` - export bb=`echo $dirshort | nawk -F"_" '{ print $2 }'` - aaupper=`echo $dirshort | nawk -F"_" '{ print toupper($1) }'` - if [ $aaupper = "EN" ] + aa="$(echo "$dirshort" | cut -d_ -f1)" + bb="$(echo "$dirshort" | cut -d_ -f2)" + export aa ; export bb + aaupper=$(echo "$dirshort" | awk -F_ '{ print toupper($1) }') + if [ "$aaupper" = "EN" ] then aaupper="US" fi - if [ $aaupper = "EL" ] + if [ "$aaupper" = "EL" ] then aaupper="GR" fi - if [ $bb = "EG" ] + if [ "${bb}" = "EG" ] then aaupper="SA" fi - if [ $bb = "IQ" ] + if [ "${bb}" = "IQ" ] then aaupper="SA" fi - bblower=`echo $dirshort | nawk -F"_" '{ print tolower($2) }'` + bblower=$(echo "$dirshort" | awk -F_ '{ print tolower($2) }') - echo "***** Process language "$aa"_"$bb - if [ "$aa" != "$bblower" -a "$dirshort" != "en_US" ] + echo "***** Process language '${aa}_${bb}'" + if [ "$aa" != "$bblower" ] && [ "$dirshort" != "en_US" ] then - reflang="htdocs/langs/"$aa"_"$aaupper - echo $reflang" "$aa"_"$bb != $aa"_"$aaupper + reflang="htdocs/langs/${aa}_$aaupper" + echo "$reflang '${aa}_${bb}' != '${aa}_$aaupper'" # If $reflang is a main language to use to sanitize the alternative file - if [ -d $reflang ] + if [ -d "$reflang" ] then - if [ $aa"_"$bb != $aa"_"$aaupper ] + if [ "${aa}_${bb}" != "${aa}_$aaupper" ] then - echo "***** Search original into "$reflang - echo $dirshort is an alternative language of $reflang - echo ./dev/translation/strip_language_file.php $aa"_"$aaupper $aa"_"$bb $2 - ./dev/translation/strip_language_file.php $aa"_"$aaupper $aa"_"$bb $2 - for fic in `ls htdocs/langs/${aa}_${bb}/*.delta`; do f=`echo $fic | sed -e 's/\.delta//'`; echo $f; mv $f.delta $f; done - for fic in `ls htdocs/langs/${aa}_${bb}/*.lang`; - do f=`cat $fic | wc -l`; + echo "***** Search original in $reflang" + echo "$dirshort is an alternative language of $reflang" + echo "./dev/translation/strip_language_file.php '${aa}_$aaupper' '${aa}_${bb}' '$LANGFILE'" + RESULT=$(./dev/translation/strip_language_file.php "${aa}_${aaupper}" "${aa}_${bb}" "$LANGFILE") + changed=0 + for fic in htdocs/langs/"${aa}_${bb}"/*.delta ; do + # No delta file found ('*' surely still present) + if [ ! -r "$fic" ] ; then break ; fi + f=${fic//\.delta/} + if diff -q "$f" "$f.delta" >/dev/null ; then + rm "$f.delta" + else + mv "$f.delta" "$f" ; changed=1 ; exit_code=1 + fi + done + [ "$changed" != "0" ] && echo "$RESULT" + for fic in htdocs/langs/"${aa}_${bb}"/*.lang ; + do f=$(wc -l < "$fic"); #echo $f lines into file $fic; - if [ $f = 1 ] + if [ "$f" = 1 ] then - echo "Only one line remaining into file '$fic', we delete it"; - rm $fic + exit_code=1 + echo "Only one line remaining in file '$fic', we delete it"; + rm "$fic" fi; done fi @@ -89,3 +117,5 @@ then fi done; fi + +exit $exit_code diff --git a/dev/tools/fixaltlanguages_pre-commit.sh b/dev/tools/fixaltlanguages_pre-commit.sh new file mode 100755 index 00000000000..0030143e4f9 --- /dev/null +++ b/dev/tools/fixaltlanguages_pre-commit.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Wrapper to run 'fixaltlanguages.sh' from pre-commit hook +# +# Copyright (C) 2024 MDW + + +# Note: regex in pre-commit based on list of fixed alt languages when run for +# all cases (or all languages with more than one code in the project). +# +# Perl script: +# +# ```perl +# use Regexp::Optimizer; +# my $o = Regexp::Optimizer->new; +# my $re= "am|ar|bn|br|bs|ca|cs|cy|da|de|el|en|es|et|eu|fa|fr|gl|he|hi" +# ."|it|ja|ka|kk|km|kn|ko|lo|ms|my|nb|ne|nl|pt|ru|sl|sq|sr|sv|ta" +# ."|tg|uk|ur|vi|zh"; +# my $newRe=$o->optimize(qr/$re/); +# print $newRe; +# ``` + +MYDIR=$(dirname "$(realpath "$0")") + +exit_code=0 +for file in "$@" ; do + if ! "${MYDIR}/fixaltlanguages.sh" fix "$file" ; then + exit_code=$? + fi +done +exit $exit_code diff --git a/dev/tools/fixduplicatelangkey.sh b/dev/tools/fixduplicatelangkey.sh index 885b2e891cc..4137829bd61 100755 --- a/dev/tools/fixduplicatelangkey.sh +++ b/dev/tools/fixduplicatelangkey.sh @@ -1,41 +1,52 @@ #!/bin/sh # Helps find duplicate translation keys in language files # -# Copyright (C) 2014 Raphaël Doursenaud - rdoursenaud@gpcsolutions.fr - -# shellcheck disable=2006,2035,2044,2061,2166,2268 +# Copyright (C) 2014 Raphaël Doursenaud +# Copyright (C) 2024 MDW +exit_code=0 # Syntax -if [ "x$1" != "xlist" -a "x$1" != "xfix" ] +if [ "$1" != "list" ] && [ "$1" != "fix" ] then echo "Detect duplicate translation keys inside a file (there is no cross file check)." echo "Usage: detectduplicatelangkey.sh (list|fix)" + exit_code=1 fi -if [ "x$1" = "xlist" ] +ACTION=$1 + +if [ "${ACTION}" = "list" ] then echo "Search duplicate keys into en_US lang files (there is no cross file check)" - for file in `find htdocs/langs/en_US -name *.lang -type f` + for file in htdocs/langs/en_US/*.lang do dupes=$( sed "s/^\s*//" "$file" | # Remove any leading whitespace sed "s/\s*\=/=/" | # Remove any whitespace before = - grep -Po "(^.*?)=" | # Non greedeely match everything before = - sed "s/\=//" | # Remove trailing = so we get the key - sort | uniq -d # Find duplicates + grep -Po "(^.*?)(?==)" | # Non greedy match everything before = + sort | uniq -d | # Find duplicates + while IFS= read -r key ; do + grep -n "^$key" "$file" | + # Format line to be recognised for code annotation by logToCs.py + echo "$file:$(cut -d ':' -f 1 | tail -n 1):error:Duplicate '$key'" + done + # awk '$0="'"$file"':"$0' # Prefix with filename (for ci) ) if [ -n "$dupes" ] then - echo "Duplicates found in $file" + exit_code=1 echo "$dupes" fi done fi # To convert -if [ "x$1" = "xfix" ] +if [ "${ACTION}" = "fix" ] then echo Feature not implemented. Please fix files manually. + exit_code=1 fi + +exit $exit_code diff --git a/dev/tools/fixduplicatelanglines.sh b/dev/tools/fixduplicatelanglines.sh index f61afbaacd2..4e87d263f15 100755 --- a/dev/tools/fixduplicatelanglines.sh +++ b/dev/tools/fixduplicatelanglines.sh @@ -1,40 +1,47 @@ -#!/bin/sh +#!/bin/bash # Recursively deduplicate file lines on a per file basis # Useful to deduplicate language files # # Needs awk 4.0 for the inplace fixing command # -# Raphaël Doursenaud - rdoursenaud@gpcsolutions.fr +# Copyright (C) 2016 Raphaël Doursenaud +# Copyright (C) 2024 MDW -# shellcheck disable=2006,2035,2044,2046,2061,2166,2268 +exit_code=0 -# Syntax -if [ "x$1" != "xlist" -a "x$1" != "xfix" ] +# Check arguments +if [ "$1" != "list" ] && [ "$1" != "fix" ] then echo "Find exact duplicated lines into file (not cross file checking)" - echo "Usage: deduplicatefilelinesrecursively.sh [list|fix]" + echo "Usage: $(basename "$0") [list|fix]" + exit_code=1 fi +ACTION=$1 + # To detect -if [ "x$1" = "xlist" ] +if [ "${ACTION}" = "list" ] || [ "${ACTION}" = "fix" ] then - echo "Search duplicate line for lang en_US" - for file in `find htdocs/langs/en_US -type f -name *.lang` + echo "Search duplicate lines for lang en_US" + echo "" + for file in htdocs/langs/en_US/*.lang do - if [ `sort "$file" | grep -v '^$' | uniq -d | wc -l` -gt 0 ] + if [ "$(sort "$file" | grep -v -P '^#?$' | uniq -d | wc -l)" -gt 0 ] then - echo "***** $file" - sort "$file" | grep -v '^$' | uniq -d + sort "$file" | grep -v -P '^#?$' | uniq -d | awk '$0="'"$file"':"$0' + exit_code=1 fi done fi # To fix -if [ "x$1" = "xfix" ] +if [ "${ACTION}" = "fix" ] then echo "Fix duplicate line for lang en_US" - for file in `find htdocs/langs/en_US -type f -name *.lang` - do + # shellcheck disable=2016 + for file in htdocs/langs/en_US/*.lang ; do awk -i inplace ' !x[$0]++' "$file" - done; + done fi + +exit $exit_code diff --git a/dev/tools/fixnotabfiles.sh b/dev/tools/fixnotabfiles.sh index 84acd9ce773..f8a383f1005 100755 --- a/dev/tools/fixnotabfiles.sh +++ b/dev/tools/fixnotabfiles.sh @@ -6,23 +6,22 @@ #------------------------------------------------------ # Usage: fixnotabfiles.sh [list|fix] #------------------------------------------------------ -# shellcheck disable=2166,2268 # Syntax -if [ "x$1" != "xlist" -a "x$1" != "xfix" ] +if [ "$1" != "list" ] && [ "$1" != "fix" ] then - echo "Detect .sh and .spec files that does not contains any tab inside" + echo "Detect .sh and .spec files that does not contain any tab" echo "Usage: fixnotabfiles.sh [list|fix]" fi -# To detec -if [ "x$1" = "xlist" ] +# List/Detect files +if [ "$1" = "list" ] then - find build \( -iname "*.sh" -o -iname "*.spec" \) -exec grep -l -P '\t' {} \; + find build \( -iname "*.sh" -o -iname "*.spec" \) -exec grep -L -P '\t' {} \; fi -# To convert -if [ "x$1" = "xfix" ] +# Fix/convert files +if [ "$1" = "fix" ] then echo Feature not implemented. Please fix files manually. fi diff --git a/dev/translation/strip_language_file.php b/dev/translation/strip_language_file.php index 3c3ad243cd2..c1f77950f41 100755 --- a/dev/translation/strip_language_file.php +++ b/dev/translation/strip_language_file.php @@ -2,6 +2,7 @@ + * Copyright (C) 2024 MDW * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -141,7 +142,7 @@ foreach ($filesToProcess as $fileToProcess) { $a = mb_split('=', trim($line), 2); if (count($a) != 2) { - print "ERROR in file $lSecondaryFile, line $cnt: " . trim($line) . "\n"; + print "File $lSecondaryFile:ERROR: ".trim($line)." in line $cnt.\n"; continue; } @@ -149,13 +150,13 @@ foreach ($filesToProcess as $fileToProcess) { // key is redundant if (array_key_exists($key, $aSecondary)) { - print "Key $key is redundant in file $lSecondaryFile (line: $cnt).\n"; + print "File $lSecondaryFile:WARNING: Key $key is redundant in line $cnt.\n"; continue; } // String has no value if ($value == '') { - print "Key $key has no value in file $lSecondaryFile (line: $cnt).\n"; + print "File $lSecondaryFile:WARNING: Key $key has no value in line: $cnt.\n"; continue; } @@ -195,7 +196,7 @@ foreach ($filesToProcess as $fileToProcess) { $a = mb_split('=', trim($line), 2); if (count($a) != 2) { - print "ERROR in file $lEnglishFile, line $cnt: " . trim($line) . "\n"; + print "File $lEnglishFile:ERROR: ".trim($line)." in line $cnt.\n"; continue; } @@ -203,13 +204,13 @@ foreach ($filesToProcess as $fileToProcess) { // key is redundant if (array_key_exists($key, $aEnglish)) { - print "Key $key is redundant in file $lEnglishFile (line: $cnt).\n"; + print "File $lEnglishFile:WARNING: Key $key is redundant in line $cnt.\n"; continue; } // String has no value if ($value == '') { - print "Key $key has no value in file $lEnglishFile (line: $cnt).\n"; + print "File $lEnglishFile:WARNING: Key $key has no value in line $cnt.\n"; continue; } @@ -238,7 +239,7 @@ foreach ($filesToProcess as $fileToProcess) { if ($handle = fopen($lPrimaryFile, 'r')) { if (! $oh = fopen($output, 'w')) { - print "ERROR in writing to file ".$output."\n"; + print "ERROR writing to file ".$output."\n"; exit; } @@ -264,7 +265,7 @@ foreach ($filesToProcess as $fileToProcess) { $a = mb_split('=', trim($line), 2); if (count($a) != 2) { - print "ERROR in file $lPrimaryFile, line $cnt: " . trim($line) . "\n"; + print "File $lPrimaryFile:ERROR: ".trim($line)." in line $cnt.\n"; continue; } @@ -272,15 +273,17 @@ foreach ($filesToProcess as $fileToProcess) { // key is redundant if (array_key_exists($key, $aPrimary)) { - print "Key $key is redundant in file $lPrimaryFile (line: $cnt)"; + $prefix = "File $lPrimaryFile:WARNING: Key $key is redundant"; + $postfix = " in line $cnt.\n"; + if (!empty($fileFirstFound[$key])) { - print " - Already found into ".$fileFirstFound[$key]; - print " (line: ".$lineFirstFound[$key].").\n"; + print "$prefix [Already found in '".$fileFirstFound[$key]; + print "' (line: ".$lineFirstFound[$key].")]$postfix";";"; } else { $fileFirstFound[$key] = $fileToProcess; $lineFirstFound[$key] = $cnt; - print " - Already found into main file.\n"; + print "$prefix [Already found in main file]$postfix"; } continue; } else { @@ -290,7 +293,7 @@ foreach ($filesToProcess as $fileToProcess) { // String has no value if ($value == '') { - print "Key $key has no value in file $lPrimaryFile (line: $cnt).\n"; + print "File $lPrimaryFile:WARNING: Key $key has no value in line $cnt.\n"; continue; } @@ -315,7 +318,7 @@ foreach ($filesToProcess as $fileToProcess) { if ((!empty($aSecondary[$key]) && $aSecondary[$key] != $aPrimary[$key] && !empty($aEnglish[$key]) && $aSecondary[$key] != $aEnglish[$key]) || in_array($key, $arrayofkeytoalwayskeep) || preg_match('/^FormatDate/', $key) || preg_match('/^FormatHour/', $key) - ) { + ) { //print "Key $key differs (aSecondary=".$aSecondary[$key].", aPrimary=".$aPrimary[$key].", aEnglish=".$aEnglish[$key].") so we add it into new secondary language (line: $cnt).\n"; fwrite($oh, $key."=".(empty($aSecondary[$key]) ? $aPrimary[$key] : $aSecondary[$key])."\n"); }