Wavyzz/dolibarr-fork
2
0
Fork 0
forked from Wavyzz/dolibarr
Code Pull Requests Activity

Qual Language scripts fixed/updated (#28101)

Browse Source 
* Qual: Fix/extend language scripts, integrate in pre-commit

Updated scripts to comply with shellscript.
Integrated one in pre-commit that seems useful.

* Qual: Find duplicate lang keys

* Add line numbers (to allow annotation)

* Fix fixaltlanguages.sh and add to pre-commit

* Make historical language checks 'manual' in 'pre-commit'
This commit is contained in:
MDW
2024-02-10 17:26:34 +01:00
committed by GitHub
parent ad759c4826
commit 16f723f0ba
7 changed files with 205 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
.pre-commit-config.yaml
View File

@@ -7,11 +7,19 @@ repos:
hooks: hooks:
- id: no-commit-to-branch - id: no-commit-to-branch
args: [--branch, develop, --pattern, \d+.0] args: [--branch, develop, --pattern, \d+.0]
- id: check-xml
exclude: |
(?x)^(htdocs/includes/.*)$
- id: check-yaml - id: check-yaml
args: [--unsafe] args: [--unsafe]
- id: check-json - id: check-json
- id: mixed-line-ending - id: mixed-line-ending
exclude: (?x)^(htdocs/includes/tecnickcom/tcpdf/fonts/.*)$ # alternative for dev/tools/fixdosfiles.sh
exclude: |
(?x)^(htdocs/includes/tecnickcom/tcpdf/fonts/.*
|.*/CRLF.*.php # Files in swiftmailer
)$
args: [--fix=lf]
- id: trailing-whitespace - id: trailing-whitespace
exclude_types: [markdown] exclude_types: [markdown]
- id: end-of-file-fixer - id: end-of-file-fixer
@@ -41,8 +49,8 @@ repos:
# ```shell # ```shell
# #!/bin/bash # #!/bin/bash
# MYDIR=$(dirname "$0") # MYDIR=$(dirname "$0")
# CHANGED_INTERNALS=$(git diff --name-only | grep -v includes) # git diff HEAD --name-only | grep -v includes | \
# "$MYDIR/dev/tools/updatelicense.php" $CHANGED_INTERNALS # xargs "$MYDIR/dev/tools/updatelicense.php"
# ``` # ```
- repo: local - repo: local
hooks: hooks:
@@ -51,6 +59,30 @@ repos:
language: system language: system
entry: bash -c '[ ! -x local.sh ] || ./local.sh' entry: bash -c '[ ! -x local.sh ] || ./local.sh'
pass_filenames: false pass_filenames: false
- id: duplicate-lang-lines
stages: [manual]
name: Find duplicate language lines
files: (?x)^(htdocs/langs/en_US/.*\.lang)
language: script
entry: ./dev/tools/fixduplicatelanglines.sh
pass_filenames: false
args: [list]
- id: duplicate-lang-keys
stages: [manual]
name: Find duplicate language keys
files: (?x)^(htdocs/langs/en_US/.*\.lang)
language: script
entry: ./dev/tools/fixduplicatelangkey.sh
pass_filenames: false
args: [list]
- id: fix-alt-languages
stages: [manual]
name: Fix alt languages
# Selection: see fixaltlanguages.sh script
files: (?x)^(htdocs/langs/(e[lnstu]|k[akmno]|s[lqrv]|b[nrs]|c[asy]|n[bel]|[ip]t|a[mr]|d[ae]|f[ar]|h[ei]|m[sy]|t[ag]|u[kr]|gl|ja|lo|ru|vi|zh)_[^/]*/.*\.lang)
language: script
entry: ./dev/tools/fixaltlanguages_pre-commit.sh
pass_filenames: true
# Check PHP syntax # Check PHP syntax
- repo: https://github.com/mdeweerd/pre-commit-php - repo: https://github.com/mdeweerd/pre-commit-php

112
dev/tools/fixaltlanguages.sh
View File

@@ -1,87 +1,115 @@
#!/bin/sh #!/bin/bash
# Recursively deduplicate file lines on a per file basis # Recursively deduplicate file lines on a per file basis
# Useful to deduplicate language files # Useful to deduplicate language files
# #
# Needs awk 4.0 for the inplace fixing command # Needs awk 4.0 for the inplace fixing command
# #
# Raphaël Doursenaud - rdoursenaud@gpcsolutions.fr # Copyright (C) 2016 Raphaël Doursenaud <rdoursenaud@gpcsolutions.fr>
# Copyright (C) 2024 MDW <mdeweerd@users.noreply.github.com>
# shellcheck disable=2006,2027,2044,2045,2046,2086,2155,2166,2268
# Syntax # Syntax
if [ "x$1" != "xlist" -a "x$1" != "xfix" ] if [ "$1" != "list" ] && [ "$1" != "fix" ]
then then
echo "Scan alternate language files and remove entries found into parent file" echo "Scan alternate language files and remove entries found in parent file"
echo "Usage: fixaltlanguages.sh (list|fix) (all|file.lang) [xx_XX]" echo "Usage: fixaltlanguages.sh (list|fix) (all|file.lang) [ll_CC]"
exit exit 1
fi fi
if [ "x$2" = "x" ] if [ "$2" = "" ]
then then
echo "Scan alternate language files and remove entries found into parent file" echo "Scan alternate language files and remove entries found in parent file"
echo "Usage: fixaltlanguages.sh (list|fix) (all|file.lang) [xx_XX]" echo "Usage: fixaltlanguages.sh (list|fix) (all|file.lang) [ll_CC]"
exit exit 1
fi
ACTION=$1
LANGFILE=$2
LANG=$3
exit_code=0
if [ -r "$LANGFILE" ] ; then
if [ "$LANG" = "" ] ; then
LANG=$(basename "$(dirname "$LANGFILE")")
fi
LANGFILE=$(basename "$LANGFILE")
fi fi
# To detect # To detect
if [ "x$1" = "xlist" ] if [ "$ACTION" = "list" ]
then then
echo Feature not available echo Feature not available
exit_code=1
fi fi
echo "$ACTION $LANGFILE $LANG"
# To fix # To fix
if [ "x$1" = "xfix" ] if [ "$ACTION" = "fix" ]
then then
for dir in `find htdocs/langs/$3* -type d` for dir in htdocs/langs/"$LANG"*/
do do
dirshort=`basename $dir` dirshort=$(basename "$dir")
#echo $dirshort # echo $dirshort
export aa=`echo $dirshort | nawk -F"_" '{ print $1 }'` aa="$(echo "$dirshort" | cut -d_ -f1)"
export bb=`echo $dirshort | nawk -F"_" '{ print $2 }'` bb="$(echo "$dirshort" | cut -d_ -f2)"
aaupper=`echo $dirshort | nawk -F"_" '{ print toupper($1) }'` export aa ; export bb
if [ $aaupper = "EN" ] aaupper=$(echo "$dirshort" | awk -F_ '{ print toupper($1) }')
if [ "$aaupper" = "EN" ]
then then
aaupper="US" aaupper="US"
fi fi
if [ $aaupper = "EL" ] if [ "$aaupper" = "EL" ]
then then
aaupper="GR" aaupper="GR"
fi fi
if [ $bb = "EG" ] if [ "${bb}" = "EG" ]
then then
aaupper="SA" aaupper="SA"
fi fi
if [ $bb = "IQ" ] if [ "${bb}" = "IQ" ]
then then
aaupper="SA" aaupper="SA"
fi fi
bblower=`echo $dirshort | nawk -F"_" '{ print tolower($2) }'` bblower=$(echo "$dirshort" | awk -F_ '{ print tolower($2) }')
echo "***** Process language "$aa"_"$bb echo "***** Process language '${aa}_${bb}'"
if [ "$aa" != "$bblower" -a "$dirshort" != "en_US" ] if [ "$aa" != "$bblower" ] && [ "$dirshort" != "en_US" ]
then then
reflang="htdocs/langs/"$aa"_"$aaupper reflang="htdocs/langs/${aa}_$aaupper"
echo $reflang" "$aa"_"$bb != $aa"_"$aaupper echo "$reflang '${aa}_${bb}' != '${aa}_$aaupper'"
# If $reflang is a main language to use to sanitize the alternative file # If $reflang is a main language to use to sanitize the alternative file
if [ -d $reflang ] if [ -d "$reflang" ]
then then
if [ $aa"_"$bb != $aa"_"$aaupper ] if [ "${aa}_${bb}" != "${aa}_$aaupper" ]
then then
echo "***** Search original into "$reflang echo "***** Search original in $reflang"
echo $dirshort is an alternative language of $reflang echo "$dirshort is an alternative language of $reflang"
echo ./dev/translation/strip_language_file.php $aa"_"$aaupper $aa"_"$bb $2 echo "./dev/translation/strip_language_file.php '${aa}_$aaupper' '${aa}_${bb}' '$LANGFILE'"
./dev/translation/strip_language_file.php $aa"_"$aaupper $aa"_"$bb $2 RESULT=$(./dev/translation/strip_language_file.php "${aa}_${aaupper}" "${aa}_${bb}" "$LANGFILE")
for fic in `ls htdocs/langs/${aa}_${bb}/*.delta`; do f=`echo $fic | sed -e 's/\.delta//'`; echo $f; mv $f.delta $f; done changed=0
for fic in `ls htdocs/langs/${aa}_${bb}/*.lang`; for fic in htdocs/langs/"${aa}_${bb}"/*.delta ; do
do f=`cat $fic | wc -l`; # No delta file found ('*' surely still present)
if [ ! -r "$fic" ] ; then break ; fi
f=${fic//\.delta/}
if diff -q "$f" "$f.delta" >/dev/null ; then
rm "$f.delta"
else
mv "$f.delta" "$f" ; changed=1 ; exit_code=1
fi
done
[ "$changed" != "0" ] && echo "$RESULT"
for fic in htdocs/langs/"${aa}_${bb}"/*.lang ;
do f=$(wc -l < "$fic");
#echo $f lines into file $fic; #echo $f lines into file $fic;
if [ $f = 1 ] if [ "$f" = 1 ]
then then
echo "Only one line remaining into file '$fic', we delete it"; exit_code=1
rm $fic echo "Only one line remaining in file '$fic', we delete it";
rm "$fic"
fi; fi;
done done
fi fi
@@ -89,3 +117,5 @@ then
fi fi
done; done;
fi fi
exit $exit_code

30
dev/tools/fixaltlanguages_pre-commit.sh Executable file
View File

@@ -0,0 +1,30 @@
#!/bin/bash
# Wrapper to run 'fixaltlanguages.sh' from pre-commit hook
#
# Copyright (C) 2024 MDW <mdeweerd@users.noreply.github.com>
# Note: regex in pre-commit based on list of fixed alt languages when run for
# all cases (or all languages with more than one code in the project).
#
# Perl script:
#
# ```perl
# use Regexp::Optimizer;
# my $o = Regexp::Optimizer->new;
# my $re= "am|ar|bn|br|bs|ca|cs|cy|da|de|el|en|es|et|eu|fa|fr|gl|he|hi"
# ."|it|ja|ka|kk|km|kn|ko|lo|ms|my|nb|ne|nl|pt|ru|sl|sq|sr|sv|ta"
# ."|tg|uk|ur|vi|zh";
# my $newRe=$o->optimize(qr/$re/);
# print $newRe;
# ```
MYDIR=$(dirname "$(realpath "$0")")
exit_code=0
for file in "$@" ; do
if ! "${MYDIR}/fixaltlanguages.sh" fix "$file" ; then
exit_code=$?
fi
done
exit $exit_code

33
dev/tools/fixduplicatelangkey.sh
View File

@@ -1,41 +1,52 @@
#!/bin/sh #!/bin/sh
# Helps find duplicate translation keys in language files # Helps find duplicate translation keys in language files
# #
# Copyright (C) 2014 Raphaël Doursenaud - rdoursenaud@gpcsolutions.fr # Copyright (C) 2014 Raphaël Doursenaud <rdoursenaud@gpcsolutions.fr>
# Copyright (C) 2024 MDW <mdeweerd@users.noreply.github.com>
# shellcheck disable=2006,2035,2044,2061,2166,2268
exit_code=0
# Syntax # Syntax
if [ "x$1" != "xlist" -a "x$1" != "xfix" ] if [ "$1" != "list" ] && [ "$1" != "fix" ]
then then
echo "Detect duplicate translation keys inside a file (there is no cross file check)." echo "Detect duplicate translation keys inside a file (there is no cross file check)."
echo "Usage: detectduplicatelangkey.sh (list|fix)" echo "Usage: detectduplicatelangkey.sh (list|fix)"
exit_code=1
fi fi
if [ "x$1" = "xlist" ] ACTION=$1
if [ "${ACTION}" = "list" ]
then then
echo "Search duplicate keys into en_US lang files (there is no cross file check)" echo "Search duplicate keys into en_US lang files (there is no cross file check)"
for file in `find htdocs/langs/en_US -name *.lang -type f` for file in htdocs/langs/en_US/*.lang
do do
dupes=$( dupes=$(
sed "s/^\s*//" "$file" | # Remove any leading whitespace sed "s/^\s*//" "$file" | # Remove any leading whitespace
sed "s/\s*\=/=/" | # Remove any whitespace before = sed "s/\s*\=/=/" | # Remove any whitespace before =
grep -Po "(^.*?)=" | # Non greedeely match everything before = grep -Po "(^.*?)(?==)" | # Non greedy match everything before =
sed "s/\=//" | # Remove trailing = so we get the key sort | uniq -d | # Find duplicates
sort | uniq -d # Find duplicates while IFS= read -r key ; do
grep -n "^$key" "$file" |
# Format line to be recognised for code annotation by logToCs.py
echo "$file:$(cut -d ':' -f 1 | tail -n 1):error:Duplicate '$key'"
done
# awk '$0="'"$file"':"$0' # Prefix with filename (for ci)
) )
if [ -n "$dupes" ] if [ -n "$dupes" ]
then then
echo "Duplicates found in $file" exit_code=1
echo "$dupes" echo "$dupes"
fi fi
done done
fi fi
# To convert # To convert
if [ "x$1" = "xfix" ] if [ "${ACTION}" = "fix" ]
then then
echo Feature not implemented. Please fix files manually. echo Feature not implemented. Please fix files manually.
exit_code=1
fi fi
exit $exit_code

39
dev/tools/fixduplicatelanglines.sh
View File

@@ -1,40 +1,47 @@
#!/bin/sh #!/bin/bash
# Recursively deduplicate file lines on a per file basis # Recursively deduplicate file lines on a per file basis
# Useful to deduplicate language files # Useful to deduplicate language files
# #
# Needs awk 4.0 for the inplace fixing command # Needs awk 4.0 for the inplace fixing command
# #
# Raphaël Doursenaud - rdoursenaud@gpcsolutions.fr # Copyright (C) 2016 Raphaël Doursenaud <rdoursenaud@gpcsolutions.fr>
# Copyright (C) 2024 MDW <mdeweerd@users.noreply.github.com>
# shellcheck disable=2006,2035,2044,2046,2061,2166,2268 exit_code=0
# Syntax # Check arguments
if [ "x$1" != "xlist" -a "x$1" != "xfix" ] if [ "$1" != "list" ] && [ "$1" != "fix" ]
then then
echo "Find exact duplicated lines into file (not cross file checking)" echo "Find exact duplicated lines into file (not cross file checking)"
echo "Usage: deduplicatefilelinesrecursively.sh [list|fix]" echo "Usage: $(basename "$0") [list|fix]"
exit_code=1
fi fi
ACTION=$1
# To detect # To detect
if [ "x$1" = "xlist" ] if [ "${ACTION}" = "list" ] || [ "${ACTION}" = "fix" ]
then then
echo "Search duplicate line for lang en_US" echo "Search duplicate lines for lang en_US"
for file in `find htdocs/langs/en_US -type f -name *.lang` echo ""
for file in htdocs/langs/en_US/*.lang
do do
if [ `sort "$file" | grep -v '^$' | uniq -d | wc -l` -gt 0 ] if [ "$(sort "$file" | grep -v -P '^#?$' | uniq -d | wc -l)" -gt 0 ]
then then
echo "***** $file" sort "$file" | grep -v -P '^#?$' | uniq -d | awk '$0="'"$file"':"$0'
sort "$file" | grep -v '^$' | uniq -d exit_code=1
fi fi
done done
fi fi
# To fix # To fix
if [ "x$1" = "xfix" ] if [ "${ACTION}" = "fix" ]
then then
echo "Fix duplicate line for lang en_US" echo "Fix duplicate line for lang en_US"
for file in `find htdocs/langs/en_US -type f -name *.lang` # shellcheck disable=2016
do for file in htdocs/langs/en_US/*.lang ; do
awk -i inplace ' !x[$0]++' "$file" awk -i inplace ' !x[$0]++' "$file"
done; done
fi fi
exit $exit_code

15
dev/tools/fixnotabfiles.sh
View File

@@ -6,23 +6,22 @@
#------------------------------------------------------ #------------------------------------------------------
# Usage: fixnotabfiles.sh [list|fix] # Usage: fixnotabfiles.sh [list|fix]
#------------------------------------------------------ #------------------------------------------------------
# shellcheck disable=2166,2268
# Syntax # Syntax
if [ "x$1" != "xlist" -a "x$1" != "xfix" ] if [ "$1" != "list" ] && [ "$1" != "fix" ]
then then
echo "Detect .sh and .spec files that does not contains any tab inside" echo "Detect .sh and .spec files that does not contain any tab"
echo "Usage: fixnotabfiles.sh [list|fix]" echo "Usage: fixnotabfiles.sh [list|fix]"
fi fi
# To detec # List/Detect files
if [ "x$1" = "xlist" ] if [ "$1" = "list" ]
then then
find build \( -iname "*.sh" -o -iname "*.spec" \) -exec grep -l -P '\t' {} \; find build \( -iname "*.sh" -o -iname "*.spec" \) -exec grep -L -P '\t' {} \;
fi fi
# To convert # Fix/convert files
if [ "x$1" = "xfix" ] if [ "$1" = "fix" ]
then then
echo Feature not implemented. Please fix files manually. echo Feature not implemented. Please fix files manually.
fi fi

29
dev/translation/strip_language_file.php
View File

@@ -2,6 +2,7 @@
<?php <?php
/* Copyright (C) 2014 by FromDual GmbH, licensed under GPL v2 /* Copyright (C) 2014 by FromDual GmbH, licensed under GPL v2
* Copyright (C) 2014 Laurent Destailleur <eldy@users.sourceforge.net> * Copyright (C) 2014 Laurent Destailleur <eldy@users.sourceforge.net>
* Copyright (C) 2024 MDW <mdeweerd@users.noreply.github.com>
* *
* This program is free software; you can redistribute it and/or modify * This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by * it under the terms of the GNU General Public License as published by
@@ -141,7 +142,7 @@ foreach ($filesToProcess as $fileToProcess) {
$a = mb_split('=', trim($line), 2); $a = mb_split('=', trim($line), 2);
if (count($a) != 2) { if (count($a) != 2) {
print "ERROR in file $lSecondaryFile, line $cnt: " . trim($line) . "\n"; print "File $lSecondaryFile:ERROR: ".trim($line)." in line $cnt.\n";
continue; continue;
} }
@@ -149,13 +150,13 @@ foreach ($filesToProcess as $fileToProcess) {
// key is redundant // key is redundant
if (array_key_exists($key, $aSecondary)) { if (array_key_exists($key, $aSecondary)) {
print "Key $key is redundant in file $lSecondaryFile (line: $cnt).\n"; print "File $lSecondaryFile:WARNING: Key $key is redundant in line $cnt.\n";
continue; continue;
} }
// String has no value // String has no value
if ($value == '') { if ($value == '') {
print "Key $key has no value in file $lSecondaryFile (line: $cnt).\n"; print "File $lSecondaryFile:WARNING: Key $key has no value in line: $cnt.\n";
continue; continue;
} }
@@ -195,7 +196,7 @@ foreach ($filesToProcess as $fileToProcess) {
$a = mb_split('=', trim($line), 2); $a = mb_split('=', trim($line), 2);
if (count($a) != 2) { if (count($a) != 2) {
print "ERROR in file $lEnglishFile, line $cnt: " . trim($line) . "\n"; print "File $lEnglishFile:ERROR: ".trim($line)." in line $cnt.\n";
continue; continue;
} }
@@ -203,13 +204,13 @@ foreach ($filesToProcess as $fileToProcess) {
// key is redundant // key is redundant
if (array_key_exists($key, $aEnglish)) { if (array_key_exists($key, $aEnglish)) {
print "Key $key is redundant in file $lEnglishFile (line: $cnt).\n"; print "File $lEnglishFile:WARNING: Key $key is redundant in line $cnt.\n";
continue; continue;
} }
// String has no value // String has no value
if ($value == '') { if ($value == '') {
print "Key $key has no value in file $lEnglishFile (line: $cnt).\n"; print "File $lEnglishFile:WARNING: Key $key has no value in line $cnt.\n";
continue; continue;
} }
@@ -238,7 +239,7 @@ foreach ($filesToProcess as $fileToProcess) {
if ($handle = fopen($lPrimaryFile, 'r')) { if ($handle = fopen($lPrimaryFile, 'r')) {
if (! $oh = fopen($output, 'w')) { if (! $oh = fopen($output, 'w')) {
print "ERROR in writing to file ".$output."\n"; print "ERROR writing to file ".$output."\n";
exit; exit;
} }
@@ -264,7 +265,7 @@ foreach ($filesToProcess as $fileToProcess) {
$a = mb_split('=', trim($line), 2); $a = mb_split('=', trim($line), 2);
if (count($a) != 2) { if (count($a) != 2) {
print "ERROR in file $lPrimaryFile, line $cnt: " . trim($line) . "\n"; print "File $lPrimaryFile:ERROR: ".trim($line)." in line $cnt.\n";
continue; continue;
} }
@@ -272,15 +273,17 @@ foreach ($filesToProcess as $fileToProcess) {
// key is redundant // key is redundant
if (array_key_exists($key, $aPrimary)) { if (array_key_exists($key, $aPrimary)) {
print "Key $key is redundant in file $lPrimaryFile (line: $cnt)"; $prefix = "File $lPrimaryFile:WARNING: Key $key is redundant";
$postfix = " in line $cnt.\n";
if (!empty($fileFirstFound[$key])) { if (!empty($fileFirstFound[$key])) {
print " - Already found into ".$fileFirstFound[$key]; print "$prefix [Already found in '".$fileFirstFound[$key];
print " (line: ".$lineFirstFound[$key].").\n"; print "' (line: ".$lineFirstFound[$key].")]$postfix";";";
} else { } else {
$fileFirstFound[$key] = $fileToProcess; $fileFirstFound[$key] = $fileToProcess;
$lineFirstFound[$key] = $cnt; $lineFirstFound[$key] = $cnt;
print " - Already found into main file.\n"; print "$prefix [Already found in main file]$postfix";
} }
continue; continue;
} else { } else {
@@ -290,7 +293,7 @@ foreach ($filesToProcess as $fileToProcess) {
// String has no value // String has no value
if ($value == '') { if ($value == '') {
print "Key $key has no value in file $lPrimaryFile (line: $cnt).\n"; print "File $lPrimaryFile:WARNING: Key $key has no value in line $cnt.\n";
continue; continue;
} }