Wavyzz/dolibarr-fork
2
0
Fork 0
forked from Wavyzz/dolibarr
Code Pull Requests Activity

Qual Language scripts fixed/updated (#28101)

Browse Source 
* Qual: Fix/extend language scripts, integrate in pre-commit

Updated scripts to comply with shellscript.
Integrated one in pre-commit that seems useful.

* Qual: Find duplicate lang keys

* Add line numbers (to allow annotation)

* Fix fixaltlanguages.sh and add to pre-commit

* Make historical language checks 'manual' in 'pre-commit'
This commit is contained in:
MDW
2024-02-10 17:26:34 +01:00
committed by GitHub
parent ad759c4826
commit 16f723f0ba
7 changed files with 205 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
.pre-commit-config.yaml
View File

@@ -7,11 +7,19 @@ repos:
hooks:
- id: no-commit-to-branch
args: [--branch, develop, --pattern, \d+.0]
- id: check-xml
exclude: |
(?x)^(htdocs/includes/.*)$
- id: check-yaml
args: [--unsafe]
- id: check-json
- id: mixed-line-ending
exclude: (?x)^(htdocs/includes/tecnickcom/tcpdf/fonts/.*)$
# alternative for dev/tools/fixdosfiles.sh
exclude: |
(?x)^(htdocs/includes/tecnickcom/tcpdf/fonts/.*
|.*/CRLF.*.php # Files in swiftmailer
)$
args: [--fix=lf]
- id: trailing-whitespace
exclude_types: [markdown]
- id: end-of-file-fixer
@@ -41,8 +49,8 @@ repos:
# ```shell
# #!/bin/bash
# MYDIR=$(dirname "$0")
# CHANGED_INTERNALS=$(git diff --name-only | grep -v includes)
# "$MYDIR/dev/tools/updatelicense.php" $CHANGED_INTERNALS
# git diff HEAD --name-only | grep -v includes | \
# xargs "$MYDIR/dev/tools/updatelicense.php"
# ```
- repo: local
hooks:
@@ -51,6 +59,30 @@ repos:
language: system
entry: bash -c '[ ! -x local.sh ] || ./local.sh'
pass_filenames: false
- id: duplicate-lang-lines
stages: [manual]
name: Find duplicate language lines
files: (?x)^(htdocs/langs/en_US/.*\.lang)
language: script
entry: ./dev/tools/fixduplicatelanglines.sh
pass_filenames: false
args: [list]
- id: duplicate-lang-keys
stages: [manual]
name: Find duplicate language keys
files: (?x)^(htdocs/langs/en_US/.*\.lang)
language: script
entry: ./dev/tools/fixduplicatelangkey.sh
pass_filenames: false
args: [list]
- id: fix-alt-languages
stages: [manual]
name: Fix alt languages
# Selection: see fixaltlanguages.sh script
files: (?x)^(htdocs/langs/(e[lnstu]|k[akmno]|s[lqrv]|b[nrs]|c[asy]|n[bel]|[ip]t|a[mr]|d[ae]|f[ar]|h[ei]|m[sy]|t[ag]|u[kr]|gl|ja|lo|ru|vi|zh)_[^/]*/.*\.lang)
language: script
entry: ./dev/tools/fixaltlanguages_pre-commit.sh
pass_filenames: true
# Check PHP syntax
- repo: https://github.com/mdeweerd/pre-commit-php

112
dev/tools/fixaltlanguages.sh
View File

@@ -1,87 +1,115 @@
#!/bin/sh
#!/bin/bash
# Recursively deduplicate file lines on a per file basis
# Useful to deduplicate language files
#
# Needs awk 4.0 for the inplace fixing command
#
# Raphaël Doursenaud - rdoursenaud@gpcsolutions.fr
# shellcheck disable=2006,2027,2044,2045,2046,2086,2155,2166,2268
# Copyright (C) 2016 Raphaël Doursenaud <rdoursenaud@gpcsolutions.fr>
# Copyright (C) 2024 MDW <mdeweerd@users.noreply.github.com>
# Syntax
if [ "x$1" != "xlist" -a "x$1" != "xfix" ]
if [ "$1" != "list" ] && [ "$1" != "fix" ]
then
echo "Scan alternate language files and remove entries found into parent file"
echo "Usage: fixaltlanguages.sh (list|fix) (all|file.lang) [xx_XX]"
exit
echo "Scan alternate language files and remove entries found in parent file"
echo "Usage: fixaltlanguages.sh (list|fix) (all|file.lang) [ll_CC]"
exit 1
fi
if [ "x$2" = "x" ]
if [ "$2" = "" ]
then
echo "Scan alternate language files and remove entries found into parent file"
echo "Usage: fixaltlanguages.sh (list|fix) (all|file.lang) [xx_XX]"
exit
echo "Scan alternate language files and remove entries found in parent file"
echo "Usage: fixaltlanguages.sh (list|fix) (all|file.lang) [ll_CC]"
exit 1
fi
ACTION=$1
LANGFILE=$2
LANG=$3
exit_code=0
if [ -r "$LANGFILE" ] ; then
if [ "$LANG" = "" ] ; then
LANG=$(basename "$(dirname "$LANGFILE")")
fi
LANGFILE=$(basename "$LANGFILE")
fi
# To detect
if [ "x$1" = "xlist" ]
if [ "$ACTION" = "list" ]
then
echo Feature not available
exit_code=1
fi
echo "$ACTION $LANGFILE $LANG"
# To fix
if [ "x$1" = "xfix" ]
if [ "$ACTION" = "fix" ]
then
for dir in `find htdocs/langs/$3* -type d`
for dir in htdocs/langs/"$LANG"*/
do
dirshort=`basename $dir`
dirshort=$(basename "$dir")
#echo $dirshort
# echo $dirshort
export aa=`echo $dirshort | nawk -F"_" '{ print $1 }'`
export bb=`echo $dirshort | nawk -F"_" '{ print $2 }'`
aaupper=`echo $dirshort | nawk -F"_" '{ print toupper($1) }'`
if [ $aaupper = "EN" ]
aa="$(echo "$dirshort" | cut -d_ -f1)"
bb="$(echo "$dirshort" | cut -d_ -f2)"
export aa ; export bb
aaupper=$(echo "$dirshort" | awk -F_ '{ print toupper($1) }')
if [ "$aaupper" = "EN" ]
then
aaupper="US"
fi
if [ $aaupper = "EL" ]
if [ "$aaupper" = "EL" ]
then
aaupper="GR"
fi
if [ $bb = "EG" ]
if [ "${bb}" = "EG" ]
then
aaupper="SA"
fi
if [ $bb = "IQ" ]
if [ "${bb}" = "IQ" ]
then
aaupper="SA"
fi
bblower=`echo $dirshort | nawk -F"_" '{ print tolower($2) }'`
bblower=$(echo "$dirshort" | awk -F_ '{ print tolower($2) }')
echo "***** Process language "$aa"_"$bb
if [ "$aa" != "$bblower" -a "$dirshort" != "en_US" ]
echo "***** Process language '${aa}_${bb}'"
if [ "$aa" != "$bblower" ] && [ "$dirshort" != "en_US" ]
then
reflang="htdocs/langs/"$aa"_"$aaupper
echo $reflang" "$aa"_"$bb != $aa"_"$aaupper
reflang="htdocs/langs/${aa}_$aaupper"
echo "$reflang '${aa}_${bb}' != '${aa}_$aaupper'"
# If $reflang is a main language to use to sanitize the alternative file
if [ -d $reflang ]
if [ -d "$reflang" ]
then
if [ $aa"_"$bb != $aa"_"$aaupper ]
if [ "${aa}_${bb}" != "${aa}_$aaupper" ]
then
echo "***** Search original into "$reflang
echo $dirshort is an alternative language of $reflang
echo ./dev/translation/strip_language_file.php $aa"_"$aaupper $aa"_"$bb $2
./dev/translation/strip_language_file.php $aa"_"$aaupper $aa"_"$bb $2
for fic in `ls htdocs/langs/${aa}_${bb}/*.delta`; do f=`echo $fic | sed -e 's/\.delta//'`; echo $f; mv $f.delta $f; done
for fic in `ls htdocs/langs/${aa}_${bb}/*.lang`;
do f=`cat $fic | wc -l`;
echo "***** Search original in $reflang"
echo "$dirshort is an alternative language of $reflang"
echo "./dev/translation/strip_language_file.php '${aa}_$aaupper' '${aa}_${bb}' '$LANGFILE'"
RESULT=$(./dev/translation/strip_language_file.php "${aa}_${aaupper}" "${aa}_${bb}" "$LANGFILE")
changed=0
for fic in htdocs/langs/"${aa}_${bb}"/*.delta ; do
# No delta file found ('*' surely still present)
if [ ! -r "$fic" ] ; then break ; fi
f=${fic//\.delta/}
if diff -q "$f" "$f.delta" >/dev/null ; then
rm "$f.delta"
else
mv "$f.delta" "$f" ; changed=1 ; exit_code=1
fi
done
[ "$changed" != "0" ] && echo "$RESULT"
for fic in htdocs/langs/"${aa}_${bb}"/*.lang ;
do f=$(wc -l < "$fic");
#echo $f lines into file $fic;
if [ $f = 1 ]
if [ "$f" = 1 ]
then
echo "Only one line remaining into file '$fic', we delete it";
rm $fic
exit_code=1
echo "Only one line remaining in file '$fic', we delete it";
rm "$fic"
fi;
done
fi
@@ -89,3 +117,5 @@ then
fi
done;
fi
exit $exit_code

30
dev/tools/fixaltlanguages_pre-commit.sh Executable file
View File

@@ -0,0 +1,30 @@
#!/bin/bash
# Wrapper to run 'fixaltlanguages.sh' from pre-commit hook
#
# Copyright (C) 2024 MDW <mdeweerd@users.noreply.github.com>
# Note: regex in pre-commit based on list of fixed alt languages when run for
# all cases (or all languages with more than one code in the project).
#
# Perl script:
#
# ```perl
# use Regexp::Optimizer;
# my $o = Regexp::Optimizer->new;
# my $re= "am|ar|bn|br|bs|ca|cs|cy|da|de|el|en|es|et|eu|fa|fr|gl|he|hi"
# ."|it|ja|ka|kk|km|kn|ko|lo|ms|my|nb|ne|nl|pt|ru|sl|sq|sr|sv|ta"
# ."|tg|uk|ur|vi|zh";
# my $newRe=$o->optimize(qr/$re/);
# print $newRe;
# ```
MYDIR=$(dirname "$(realpath "$0")")
exit_code=0
for file in "$@" ; do
if ! "${MYDIR}/fixaltlanguages.sh" fix "$file" ; then
exit_code=$?
fi
done
exit $exit_code

33
dev/tools/fixduplicatelangkey.sh
View File

@@ -1,41 +1,52 @@
#!/bin/sh
# Helps find duplicate translation keys in language files
#
# Copyright (C) 2014 Raphaël Doursenaud - rdoursenaud@gpcsolutions.fr
# shellcheck disable=2006,2035,2044,2061,2166,2268
# Copyright (C) 2014 Raphaël Doursenaud <rdoursenaud@gpcsolutions.fr>
# Copyright (C) 2024 MDW <mdeweerd@users.noreply.github.com>
exit_code=0
# Syntax
if [ "x$1" != "xlist" -a "x$1" != "xfix" ]
if [ "$1" != "list" ] && [ "$1" != "fix" ]
then
echo "Detect duplicate translation keys inside a file (there is no cross file check)."
echo "Usage: detectduplicatelangkey.sh (list|fix)"
exit_code=1
fi
if [ "x$1" = "xlist" ]
ACTION=$1
if [ "${ACTION}" = "list" ]
then
echo "Search duplicate keys into en_US lang files (there is no cross file check)"
for file in `find htdocs/langs/en_US -name *.lang -type f`
for file in htdocs/langs/en_US/*.lang
do
dupes=$(
sed "s/^\s*//" "$file" | # Remove any leading whitespace
sed "s/\s*\=/=/" | # Remove any whitespace before =
grep -Po "(^.*?)=" | # Non greedeely match everything before =
sed "s/\=//" | # Remove trailing = so we get the key
sort | uniq -d # Find duplicates
grep -Po "(^.*?)(?==)" | # Non greedy match everything before =
sort | uniq -d | # Find duplicates
while IFS= read -r key ; do
grep -n "^$key" "$file" |
# Format line to be recognised for code annotation by logToCs.py
echo "$file:$(cut -d ':' -f 1 | tail -n 1):error:Duplicate '$key'"
done
# awk '$0="'"$file"':"$0' # Prefix with filename (for ci)
)
if [ -n "$dupes" ]
then
echo "Duplicates found in $file"
exit_code=1
echo "$dupes"
fi
done
fi
# To convert
if [ "x$1" = "xfix" ]
if [ "${ACTION}" = "fix" ]
then
echo Feature not implemented. Please fix files manually.
exit_code=1
fi
exit $exit_code

39
dev/tools/fixduplicatelanglines.sh
View File

@@ -1,40 +1,47 @@
#!/bin/sh
#!/bin/bash
# Recursively deduplicate file lines on a per file basis
# Useful to deduplicate language files
#
# Needs awk 4.0 for the inplace fixing command
#
# Raphaël Doursenaud - rdoursenaud@gpcsolutions.fr
# Copyright (C) 2016 Raphaël Doursenaud <rdoursenaud@gpcsolutions.fr>
# Copyright (C) 2024 MDW <mdeweerd@users.noreply.github.com>
# shellcheck disable=2006,2035,2044,2046,2061,2166,2268
exit_code=0
# Syntax
if [ "x$1" != "xlist" -a "x$1" != "xfix" ]
# Check arguments
if [ "$1" != "list" ] && [ "$1" != "fix" ]
then
echo "Find exact duplicated lines into file (not cross file checking)"
echo "Usage: deduplicatefilelinesrecursively.sh [list|fix]"
echo "Usage: $(basename "$0") [list|fix]"
exit_code=1
fi
ACTION=$1
# To detect
if [ "x$1" = "xlist" ]
if [ "${ACTION}" = "list" ] || [ "${ACTION}" = "fix" ]
then
echo "Search duplicate line for lang en_US"
for file in `find htdocs/langs/en_US -type f -name *.lang`
echo "Search duplicate lines for lang en_US"
echo ""
for file in htdocs/langs/en_US/*.lang
do
if [ `sort "$file" | grep -v '^$' | uniq -d | wc -l` -gt 0 ]
if [ "$(sort "$file" | grep -v -P '^#?$' | uniq -d | wc -l)" -gt 0 ]
then
echo "***** $file"
sort "$file" | grep -v '^$' | uniq -d
sort "$file" | grep -v -P '^#?$' | uniq -d | awk '$0="'"$file"':"$0'
exit_code=1
fi
done
fi
# To fix
if [ "x$1" = "xfix" ]
if [ "${ACTION}" = "fix" ]
then
echo "Fix duplicate line for lang en_US"
for file in `find htdocs/langs/en_US -type f -name *.lang`
do
# shellcheck disable=2016
for file in htdocs/langs/en_US/*.lang ; do
awk -i inplace ' !x[$0]++' "$file"
done;
done
fi
exit $exit_code

15
dev/tools/fixnotabfiles.sh
View File

@@ -6,23 +6,22 @@
#------------------------------------------------------
# Usage: fixnotabfiles.sh [list|fix]
#------------------------------------------------------
# shellcheck disable=2166,2268
# Syntax
if [ "x$1" != "xlist" -a "x$1" != "xfix" ]
if [ "$1" != "list" ] && [ "$1" != "fix" ]
then
echo "Detect .sh and .spec files that does not contains any tab inside"
echo "Detect .sh and .spec files that does not contain any tab"
echo "Usage: fixnotabfiles.sh [list|fix]"
fi
# To detec
if [ "x$1" = "xlist" ]
# List/Detect files
if [ "$1" = "list" ]
then
find build \( -iname "*.sh" -o -iname "*.spec" \) -exec grep -l -P '\t' {} \;
find build \( -iname "*.sh" -o -iname "*.spec" \) -exec grep -L -P '\t' {} \;
fi
# To convert
if [ "x$1" = "xfix" ]
# Fix/convert files
if [ "$1" = "fix" ]
then
echo Feature not implemented. Please fix files manually.
fi

29
dev/translation/strip_language_file.php
View File

@@ -2,6 +2,7 @@
<?php
/* Copyright (C) 2014 by FromDual GmbH, licensed under GPL v2
* Copyright (C) 2014 Laurent Destailleur <eldy@users.sourceforge.net>
* Copyright (C) 2024 MDW <mdeweerd@users.noreply.github.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@@ -141,7 +142,7 @@ foreach ($filesToProcess as $fileToProcess) {
$a = mb_split('=', trim($line), 2);
if (count($a) != 2) {
print "ERROR in file $lSecondaryFile, line $cnt: " . trim($line) . "\n";
print "File $lSecondaryFile:ERROR: ".trim($line)." in line $cnt.\n";
continue;
}
@@ -149,13 +150,13 @@ foreach ($filesToProcess as $fileToProcess) {
// key is redundant
if (array_key_exists($key, $aSecondary)) {
print "Key $key is redundant in file $lSecondaryFile (line: $cnt).\n";
print "File $lSecondaryFile:WARNING: Key $key is redundant in line $cnt.\n";
continue;
}
// String has no value
if ($value == '') {
print "Key $key has no value in file $lSecondaryFile (line: $cnt).\n";
print "File $lSecondaryFile:WARNING: Key $key has no value in line: $cnt.\n";
continue;
}
@@ -195,7 +196,7 @@ foreach ($filesToProcess as $fileToProcess) {
$a = mb_split('=', trim($line), 2);
if (count($a) != 2) {
print "ERROR in file $lEnglishFile, line $cnt: " . trim($line) . "\n";
print "File $lEnglishFile:ERROR: ".trim($line)." in line $cnt.\n";
continue;
}
@@ -203,13 +204,13 @@ foreach ($filesToProcess as $fileToProcess) {
// key is redundant
if (array_key_exists($key, $aEnglish)) {
print "Key $key is redundant in file $lEnglishFile (line: $cnt).\n";
print "File $lEnglishFile:WARNING: Key $key is redundant in line $cnt.\n";
continue;
}
// String has no value
if ($value == '') {
print "Key $key has no value in file $lEnglishFile (line: $cnt).\n";
print "File $lEnglishFile:WARNING: Key $key has no value in line $cnt.\n";
continue;
}
@@ -238,7 +239,7 @@ foreach ($filesToProcess as $fileToProcess) {
if ($handle = fopen($lPrimaryFile, 'r')) {
if (! $oh = fopen($output, 'w')) {
print "ERROR in writing to file ".$output."\n";
print "ERROR writing to file ".$output."\n";
exit;
}
@@ -264,7 +265,7 @@ foreach ($filesToProcess as $fileToProcess) {
$a = mb_split('=', trim($line), 2);
if (count($a) != 2) {
print "ERROR in file $lPrimaryFile, line $cnt: " . trim($line) . "\n";
print "File $lPrimaryFile:ERROR: ".trim($line)." in line $cnt.\n";
continue;
}
@@ -272,15 +273,17 @@ foreach ($filesToProcess as $fileToProcess) {
// key is redundant
if (array_key_exists($key, $aPrimary)) {
print "Key $key is redundant in file $lPrimaryFile (line: $cnt)";
$prefix = "File $lPrimaryFile:WARNING: Key $key is redundant";
$postfix = " in line $cnt.\n";
if (!empty($fileFirstFound[$key])) {
print " - Already found into ".$fileFirstFound[$key];
print " (line: ".$lineFirstFound[$key].").\n";
print "$prefix [Already found in '".$fileFirstFound[$key];
print "' (line: ".$lineFirstFound[$key].")]$postfix";";";
} else {
$fileFirstFound[$key] = $fileToProcess;
$lineFirstFound[$key] = $cnt;
print " - Already found into main file.\n";
print "$prefix [Already found in main file]$postfix";
}
continue;
} else {
@@ -290,7 +293,7 @@ foreach ($filesToProcess as $fileToProcess) {
// String has no value
if ($value == '') {
print "Key $key has no value in file $lPrimaryFile (line: $cnt).\n";
print "File $lPrimaryFile:WARNING: Key $key has no value in line $cnt.\n";
continue;
}